#!/usr/bin/env python
#
# PyAIEngine a new generation network intrusion detection system.
#
# Copyright (C) 2013-2023  Luis Campo Giralte
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA  02110-1301, USA.
#
# Written by Luis Campo Giralte <luis.camp0.2009@gmail.com>
#
# pylint: extension-pkg-whitelist=pyaiengine
# pylint: disable=no-member
# pylint: disable=not-an-iterable
#
"""Tests for the pyaiengine python wrapper.

This file contains unit tests and functional tests that
verifies that a generated library works as expected.

Some tests are skipped due to compilation options are not set
, some external binaries are needed to carrie out
some operations or needs sudo permissions on the tests.

"""
import os
import socket
import sys
import unittest
import glob
import json
import tempfile
import inspect
import re
import string
import random
from contextlib import contextmanager
from multiprocessing import Process
import time
import datetime
import stat
import requests
import subprocess
import platform
import pyaiengine
from http.server import *
from requests.adapters import HTTPAdapter, Retry

QUEUE_DELAY = 1
API_DELAY = 1

def clean_log_files():

    for log_file in glob.glob("aiengine-*.log"):
        if os.path.isfile(log_file):
            os.remove(log_file)

def which(program):

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None


def get_available_ip_addresses() -> list:
    """Run a ifconfig to get the available IPs for some
    unit tests that depends on them."""

    ips = ["127.0.0.1"]
    ret = subprocess.run(["ifconfig"], stdout=subprocess.PIPE)
    for line in ret.stdout.decode("ascii").split("\n"):
        if line.lstrip().startswith("inet "):
            items = line.lstrip().split(" ")
            if items[1] not in ips:
                ips.append(items[1])
    return ips


def get_free_port() -> int:
    """Retrieve a port that is not in use for the tests."""

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # Let the OS give us a non used port
    sock.bind((HTTP_LISTEN_IP, 0))
    port = sock.getsockname()[1]
    sock.close()
    return port

def generate_service_code(filename, class_name, stack, port, pcapfile,
    extra_body=[], extra_dispatcher=[]):
    """This helper function generates python script code that will
    be used on the tests."""

    now = str(datetime.datetime.now())
    head = f"""# code generated {now}
# for test cases on {class_name}
import sys
import pyaiengine

stack = pyaiengine.{stack}()

stack.tcp_flows = 2048
stack.udp_flows = 1024

rman = pyaiengine.RegexManager()

# Assign to TCP and UDP traffic just for testing
stack.udp_regex_manager = rman
stack.tcp_regex_manager = rman

"""
    # Now inject on the body
    for line in extra_body:
        head += line + "\n"

    head += f"""pcapfile = \"{pcapfile}\"
with pyaiengine.PacketDispatcher(pcapfile) as pdis:
    pdis.stack = stack
    pdis.authorized_ip_address = {AUTHORIZED_HOSTS}
    pdis.http_port = {port}
"""

    for line in extra_dispatcher:
        head += "    " + line + "\n"

    head += f"""    pdis.run()

sys.exit(0)
"""
    with open(filename, "w") as fd:
        fd.write(head)


def http_server(port):

    class Server(BaseHTTPRequestHandler):
        protocol_version = "HTTP/1.1"
        def _set_response(self):
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.send_header("Connection", "keep-alive")
            self.send_header("keep-alive", "timeout=5, max=30")
            self.send_header("Content-length", 0)
            self.end_headers()

        def do_GET(self):
            self._set_response()

    server_address = ('', port)
    httpd = HTTPServer(server_address, Server)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass
    httpd.server_close()

def udp_server(port):

    with socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) as sock:

        sock.bind(("", port))

        while True:
            _ , address = sock.recvfrom(1024)
            sock.sendto(b"PONG", address)

class DatabaseTestAdaptor(pyaiengine.DatabaseAdaptor, object):
    """
    Adaptor for verify the functionality of retrieve
    information is working on the different stacks.
    """

    def __init__(self):
        self.total_inserts = 0
        self.total_updates = 0
        self.total_removes = 0
        self.lastdata = dict()
        self.all_data = dict()

    def update(self, _, data):
        self.total_updates = self.total_updates + 1
        self.all_data[self.total_updates] = data
        self.lastdata = data

    def insert(self, key):
        self.total_inserts = self.total_inserts + 1

    def remove(self, key):
        self.total_removes = self.total_removes + 1

def defined(value):
    """Easy function to verify what has been enable or disable
    with the configure script."""

    with open("../config.h") as file:
        for line in file.readlines():
            if line.startswith("#define %s" % value):
                return True
    return False

# https://stackoverflow.com/questions/4675728/redirect-stdout-to-a-file-in-python#4675744
def fileno(file_or_fd):
    descriptor = getattr(file_or_fd, "fileno", lambda: file_or_fd)()
    if not isinstance(descriptor, int):
        raise ValueError("Expected a file (`.fileno()`) or a file descriptor")
    return descriptor

@contextmanager
def stdout_redirected(to=os.devnull, stdout=None):
    """A context manager decorator for test output redirections."""

    if stdout is None:
        stdout = sys.stdout

    stdout_fd = fileno(stdout)
    # copy stdout_fd before it is overwritten
    # NOTE: `copied` is inheritable on Windows when duplicating a standard stream
    with os.fdopen(os.dup(stdout_fd), "wb") as copied:
        stdout.flush()  # flush library buffers that dup2 knows nothing about
        try:
            os.dup2(fileno(to), stdout_fd)  # $ exec >&to
        except ValueError:  # filename
            with open(to, "wb") as to_file:
                os.dup2(to_file.fileno(), stdout_fd)  # $ exec > to
        try:
            yield stdout # allow code to be run with the redirected stdout
        finally:
            # restore stdout to its previous value
            #NOTE: dup2 makes stdout_fd inheritable unconditionally
            stdout.flush()
            os.dup2(copied.fileno(), stdout_fd)  # $ exec >&copied

class StackLanTests(unittest.TestCase):
    """
    Basic tests cases for LAN environments.
    """

    def setUp(self):
        self.stack = pyaiengine.StackLan()
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0
        self.ip_called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile, pcapfilter=""):
        """Generic method for inject the pcapfiles."""

        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            if pcapfilter:
                pdis.pcap_filter = pcapfilter
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """Create a regex for netbios and detect."""
        self.stack.link_layer_tag = "vlan"

        rman = pyaiengine.RegexManager()

        self.assertEqual(sys.getrefcount(rman), 2)
        reg = pyaiengine.Regex("netbios", "CACACACA")
        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman
        self.assertEqual(sys.getrefcount(rman), 3)

        self.stack.mode = "nids"

        self.assertEqual(self.stack.mode, "nids")
        self.assertEqual(self.stack.stats_level, 0)

        self.stack.mode = "change"

        self.assertEqual(self.stack.mode, "nids")

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(self.stack.udp_regex_manager, rman)
        self.assertEqual(self.stack.link_layer_tag, "vlan")

        # The rm is plugged to the UDP protocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            rman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"UDPGeneric", lines[0])

        self.stack.udp_regex_manager = None
        self.assertEqual(sys.getrefcount(rman), 3)

        # Test the output and the existance of the function
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            lines = file.readlines()
            self.assertEqual(len(lines), 8)
            self.assertIn(b"[131.151.104.96:137]:17:[131.151.107.255:137]", lines[6])
            self.assertIn(b"State", lines[5])
            self.assertIn(b"comatose", lines[6])

    def test02(self):
        """Verify that None is working on the udpregexmanager."""
        self.stack.link_layer_tag = "vlan"

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("netbios", "CACACACA")
        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman

        # The rman is plugged to the UDP protocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            rman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"UDPGeneric", lines[0])


        self.stack.udp_regex_manager = None

        # The rman is plugged to the UDP protocol
        self.assertEqual(str(rman).find("UDPGeneric"), -1)

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg.matchs, 0)
        self.assertIsNone(self.stack.udp_regex_manager)

        # Test the output and the existance of the function
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(limit=0)
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 7)

    def test03(self):
        """Create a regex for netbios with callback. """

        def callback(flow):
            self.called_callback += 1
            self.assertEqual(flow.regex.matchs, 1)
            self.assertEqual(flow.regex.name, "netbios")
            self.assertIsNotNone(flow.regex_manager)
            self.assertEqual(flow.regex_manager.name, rman.name)

        self.stack.link_layer_tag = "vlan"

        rman = pyaiengine.RegexManager("My regex manager")

        # Change the name of the regex manager
        rman.name = "My lovely name"
        self.assertEqual(rman.name, "My lovely name")

        reg1 = pyaiengine.Regex("netbios", "CACACACA")
        reg1.callback = callback
        reg2 = pyaiengine.Regex("other", "This is not on the packets")
        rman.add_regex(reg1)
        rman.add_regex(reg2)
        self.stack.udp_regex_manager = rman

        self.stack.mode = "nids"

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(self.called_callback, 1)

        # Test the output and the existance of the function
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            # The flow should not be shown
            self.stack.show_flows(l7protocol="DNS")
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 7)

        # Test the output of the method show_matched_regexs.
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            # The regex should be shown
            rman.show_matched_regexs()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 2)

        # Reset the statistics
        rman.reset()

        self.assertEqual(reg1.matchs, 0)
        self.assertEqual(reg2.matchs, 0)

    def test04(self):
        """Verify DNS and HTTP traffic."""

        self.inject("../pcapfiles/accessgoogle.pcap")

        tcp_flows = self.stack.tcp_flow_manager
        udp_flows = self.stack.udp_flow_manager

        self.assertEqual(len(tcp_flows), 1)
        self.assertEqual(len(udp_flows), 1)

        flow = [flow for flow in udp_flows][0]

        self.assertEqual(str(flow.dns_info.domain_name), "www.google.com")
        self.assertEqual(flow.dns_info.txid, 56072)
        self.assertEqual(flow.dns_info.rcode, 0)

        # Verify the properties of the flows
        self.assertEqual(str(flow.src_ip), "192.168.1.13")
        self.assertEqual(str(flow.dst_ip), "89.101.160.5")
        self.assertEqual(int(flow.src_port), 54737)
        self.assertEqual(int(flow.dst_port), 53)

        flow = [flow for flow in tcp_flows][0]

        # Read only attributes
        self.assertEqual(flow.packets_layer7, 4)
        self.assertEqual(flow.packets, 10)
        self.assertEqual(flow.bytes, 1826)
        self.assertEqual(flow.have_tag, False)

        self.assertEqual(str(flow.http_info.host_name), "www.google.com")
        self.assertEqual(flow.http_info.content_type, "text/html")

        # All the flows can not have ip_set assigned
        for flow in self.stack.tcp_flow_manager:
            self.assertIsNone(flow.ip_set)

        data = self.stack.get_cache_data("DNS", "name")
        self.assertEqual(len(data), 1)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="http")
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 8)

        total = [flow for flow in self.stack.tcp_flow_manager if flow.http_info]
        self.assertEqual(len(total), 1)

    def test05(self):
        """Verify SSL traffic."""

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(len(self.stack.tcp_flow_manager), 1)

        flow = [flow for flow in self.stack.tcp_flow_manager][0]

        self.assertEqual(str(flow.ssl_info.server_name), "0.drive.google.com")
        self.assertEqual(str(flow.ssl_info.issuer_name), "Google Internet Authority")
        self.assertEqual(format(flow.ssl_info.cipher, "#04x"), "0xc011")

        if defined("HAVE_JA3"):
            self.assertEqual(str(flow.ssl_info.fingerprint), "89d37026246d4888e78e69af4f8d1147")

        # Test the output and the existance of the function
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="ssl", limit=2)
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 8)

        data_cache = self.stack.get_cache_data("ssl", "host")
        self.assertEqual(len(data_cache), 1)

    def test06(self):
        """Verify SSL traffic with domain callback."""

        def domain_callback(_):
            self.called_callback += 1

        dom1 = pyaiengine.DomainName("Google Drive Cert", ".drive.google.com")
        dom2 = pyaiengine.DomainName("No idea", ".pepe.com")
        dom1.callback = domain_callback
        dom1.regex_manager = None
        dom1.http_uri_set = None
        dom1.http_uri_regex_manager = None

        self.assertEqual(dom1.regex_manager, None)

        dman = pyaiengine.DomainNameManager([dom1, dom2])
        dman.name = "Some name"
        self.assertEqual(sys.getrefcount(dman), 2)

        self.stack.set_domain_name_manager(dman, "SSL")
        self.assertEqual(sys.getrefcount(dman), 3)

        # the dm is plugged to the SSLProtocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"SSL", lines[0])

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(len(dman), 2)
        self.assertEqual(dom1.matchs, 1)
        self.assertEqual(dom2.matchs, 0)
        self.assertEqual(self.called_callback, 1)

        # check also the integrity of the ssl cache and counters
        ca1 = {"0.drive.google.com": 1}
        cache = self.stack.get_cache_data("SSL", "host")
        self.assertDictEqual(cache, ca1)

        counters = self.stack.get_counters("SSL")
        self.assertEqual(counters["server hellos"], 1)

        self.stack.set_domain_name_manager(None, "SSL")
        self.assertEqual(sys.getrefcount(dman), 2)

        # the dm is not plugged to the SSLProtocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertNotIn(b"SSL", lines[0])

        # Test the output of the matches domains of the DomainNameManager
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show_matched_domains()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 2)

        # Test the output of the matches domains of the DomainNameManager
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 3)

        # Reset the statistics of the DomainNameManager
        dman.reset()

        self.assertEqual(dom1.matchs, 0)
        self.assertEqual(dom2.matchs, 0)

    def test07(self):
        """Verify SSL traffic with domain callback and IPset."""

        def ipset_callback(_):
            self.ip_called_callback += 1

        def domain_callback(flow):
            self.called_callback += 1
            # Execute some of the properties of the flow
            self.assertEqual(flow.reject, False)
            self.assertEqual(flow.regex_manager, None)
            flow.accept = True
            self.assertEqual(flow.accept, True)

        ip = pyaiengine.IPSet("Specific IP address", ["74.125.24.189", "not valid"])
        self.assertEqual(sys.getrefcount(ip), 2)

        ip.add_ip_address("2274.125.24.189")

        # There is only one valid IP address
        self.assertEqual(len(ip), 1)

        ip.callback = ipset_callback
        self.assertEqual(ip.callback, ipset_callback)

        ipm = pyaiengine.IPSetManager()
        self.assertEqual(sys.getrefcount(ipm), 2)
        ipm.add_ip_set(ip)
        self.assertEqual(sys.getrefcount(ipm), 2)
        self.assertEqual(sys.getrefcount(ip), 3)

        dom = pyaiengine.DomainName("Google All", ".google.com")
        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.tcp_ip_set_manager = ipm
        self.assertEqual(sys.getrefcount(ipm), 3)

        # the ipm is plugged to the TCPProtocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            ipm.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"TCP", lines[0])

        self.stack.set_domain_name_manager(dman, "SSL")

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)
        self.assertEqual(self.ip_called_callback, 1)

        self.stack.tcp_ip_set_manager = None
        self.assertEqual(sys.getrefcount(ipm), 2)

        # the ipm is not plugged to the TCPProtocol
        self.assertEqual(str(ipm).find("TCP"), -1)

        ip.remove_ip_address("74.125.24.189")

    def test08(self):
        """Attach a database to the engine."""

        adaptor = DatabaseTestAdaptor()

        self.assertEqual(sys.getrefcount(adaptor), 2)
        self.stack.set_tcp_database_adaptor(adaptor, 16)
        self.assertEqual(sys.getrefcount(adaptor), 3)

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(adaptor.total_inserts, 1)
        self.assertEqual(adaptor.total_updates, 5)
        self.assertEqual(adaptor.total_removes, 0)

        # Verify the references
        self.stack.set_tcp_database_adaptor(None)
        self.stack.set_tcp_database_adaptor(None)
        self.stack.set_tcp_database_adaptor(None)
        self.assertEqual(sys.getrefcount(adaptor), 2)

    def test09(self):
        """Attach two databases to the engine."""

        self.stack.flows_timeout = 2

        adaptor1 = DatabaseTestAdaptor()
        adaptor2 = DatabaseTestAdaptor()

        self.assertEqual(sys.getrefcount(adaptor1), 2)
        self.assertEqual(sys.getrefcount(adaptor2), 2)

        self.stack.link_layer_tag = "vlan"
        self.stack.set_udp_database_adaptor(adaptor1, 16)

        self.assertEqual(sys.getrefcount(adaptor1), 3)
        self.assertEqual(sys.getrefcount(adaptor2), 2)

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(adaptor1.total_inserts, 1)
        self.assertEqual(adaptor1.total_updates, 1)
        self.assertEqual(adaptor1.total_removes, 0)
        self.assertEqual(adaptor2.total_inserts, 0)
        self.assertEqual(adaptor2.total_updates, 0)
        self.assertEqual(adaptor2.total_removes, 0)

        # Verify the output of adaptor
        data = json.loads(adaptor1.lastdata)
        if "info" in data:
            self.assertEqual(data["info"]["netbiosname"], "BLUMGROUP")

        cache = self.stack.get_cache_data("netbios", "name")
        self.assertIsNotNone(cache["BLUMGROUP"])

        # reset the flows
        self.stack.udp_flow_manager.flush()

        self.stack.set_udp_database_adaptor(adaptor2, 16)

        self.assertEqual(sys.getrefcount(adaptor1), 2)
        self.assertEqual(sys.getrefcount(adaptor2), 3)

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(adaptor1.total_inserts, 1)
        self.assertEqual(adaptor1.total_updates, 1)
        self.assertEqual(adaptor1.total_removes, 0)
        self.assertEqual(adaptor2.total_inserts, 1)
        self.assertEqual(adaptor2.total_updates, 1)
        self.assertEqual(adaptor2.total_removes, 0)

        self.stack.set_udp_database_adaptor(None)

        self.assertEqual(sys.getrefcount(adaptor1), 2)
        self.assertEqual(sys.getrefcount(adaptor2), 2)

    def test10(self):
        """Attach a database to the engine and domain name."""

        def domain_callback(flow):
            self.called_callback += 1
            self.assertEqual(str(flow.ssl_info.server_name), "0.drive.google.com")
            self.assertEqual(flow.l7protocol, "SSL")
            self.assertEqual(dom, flow.ssl_info.matched_domain_name)

        dom = pyaiengine.DomainName("Google All", ".google.com")

        self.assertEqual(sys.getrefcount(domain_callback), 2)
        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        self.assertEqual(sys.getrefcount(domain_callback), 3)
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "SSL")

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor, 16)

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(adaptor.total_inserts, 1)
        self.assertEqual(adaptor.total_updates, 5)
        self.assertEqual(adaptor.total_removes, 0)
        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)

        dom.callback = None
        self.assertEqual(sys.getrefcount(domain_callback), 2)

    def test11(self):
        """Verify iterators of the RegexManager."""

        regl = [pyaiengine.Regex("expression %d" % x, "some regex %d" % x) for x in range(0, 5)]

        # Add a list with regexs to the RegexManager
        rman = pyaiengine.RegexManager(regl)

        # For verify that we can iterate over the regexs
        for reg in rman:
            _ = reg

        self.assertIsNone(rman.callback)
        self.assertIsNone(self.stack.tcp_regex_manager)

        self.stack.tcp_regex_manager = rman
        self.stack.mode = "nids"

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(len(rman), 5)

        self.assertEqual(rman, self.stack.tcp_regex_manager)
        for reg in rman:
            self.assertEqual(reg.matchs, 0)

        self.stack.tcp_regex_manager = None
        # the rm is not plugged to the TCPGenericProtocol
        self.assertEqual(str(rman).find("TCPGeneric Protocol"), -1)

    @unittest.skipIf(not defined("HAVE_BLOOMFILTER"), "Test not supported")
    def test12(self):
        """Verify the IPBloomSet class."""

        have_bloom = False
        try:
            from pyaiengine import IPBloomSet
            have_bloom = True
        except ImportError:
            pass

        if have_bloom: # execute the test
            def ipset_callback(flow):
                self.ip_called_callback += 1

            ipset = IPBloomSet("Specific IP address")
            ipset.add_ip_address("74.125.24.189")
            ipset.callback = ipset_callback

            ipman = pyaiengine.IPSetManager()
            ipman.add_ip_set(ipset)

            self.stack.tcp_ip_set_manager = ipman

            self.inject("../pcapfiles/sslflow.pcap")

            self.assertEqual(self.ip_called_callback, 1)
            ipman.reset()

    def test13(self):
        """Verify all the URIs of an HTTP flow."""

        def domain_callback(flow):
            urls = ("/css/global.css?v=20121120a", "/js/jquery.hoverIntent.js",
                    "/js/ecom/ecomPlacement.js",
                    "/js/scrolldock/scrolldock.css?v=20121120a",
                    "/images_blogs/gadgetlab/2013/07/MG_9640edit-200x100.jpg",
                    "/images_blogs/underwire/2013/08/Back-In-Time-200x100.jpg",
                    "/images_blogs/thisdayintech/2013/03/set.jpg",
                    "/js/scrolldock/i/sub_righttab.gif",
                    "/images/global_header/new/Marriott_217x109.jpg",
                    "/images/global_header/subscribe/gh_flyout_failsafe.jpg",
                    "/images/global_header/new/the-connective.jpg",
                    "/images/covers/120x164.jpg",
                    "/images/subscribe/xrail_headline.gif",
                    "/images_blogs/gadgetlab/2013/08/bb10-bg.jpg",
                    "/images_blogs/autopia/2013/08/rescuer_cam06_110830-200x100.jpg",
                    "/images_blogs/wiredscience/2013/08/earth-ring-200x100.jpg",
                    "/images_blogs/underwire/2013/08/breaking-bad-small-200x100.png",
                    "/insights/wp-content/uploads/2013/08/dotcombubble_660-200x100.jpg",
                    "/geekdad/wp-content/uploads/2013/03/wreck-it-ralph-title1-200x100.png",
                    "/wiredenterprise/wp-content/uploads/2013/08/apple-logo-pixels-200x100.jpg",
                    "/images_blogs/threatlevel/2013/08/drone-w.jpg",
                    "/images_blogs/rawfile/2013/08/CirculationDesk-200x100.jpg",
                    "/images_blogs/magazine/2013/07/theoptimist_wired-200x100.jpg",
                    "/images_blogs/underwire/2013/08/Back-In-Time-w.jpg",
                    "/design/wp-content/uploads/2013/08/dyson-w.jpg",
                    "/images_blogs/threatlevel/2013/08/aaron_swartz-w.jpg",
                    "/images_blogs/threatlevel/2013/08/aaron_swartz-w.jpg",
                    "/images_blogs/wiredscience/2013/08/NegativelyRefracting-w.jpg",
                    "/images_blogs/wiredscience/2013/08/bee-w.jpg",
                    "/gadgetlab/2013/08/blackberry-failures/",
                    "/gadgetlab/wp-content/themes/wired-global/style.css?ver=20121114",
                    "/css/global.css?ver=20121114",
                    "/js/cn-fe-common/jquery-1.7.2.min.js?ver=1.7.2",
                    "/js/cn.minified.js?ver=20121114",
                    "/js/videos/MobileCompatibility.js?ver=20121114",
                    "/images_blogs/gadgetlab/2013/06/internets.png",
                    "/gadgetlab/wp-content/themes/wired-responsive/i/design-sprite.png",
                    "/images_blogs/gadgetlab/2013/08/Blackberry8820.jpg",
                    "/images_blogs/gadgetlab/2013/08/vsapple-60x60.jpg",
                    "/images_blogs/gadgetlab/2013/08/AP090714043057-60x60.jpg"
                    )
            self.called_callback += 1

            self.assertEqual(str(flow.http_info.uri) in urls, True)
            self.assertEqual(str(flow.http_info.host_name), "www.wired.com")
            self.assertEqual(flow.l7protocol, "HTTP")
            self.assertEqual(flow.http_info.matched_domain_name, dom1)

        dom1 = pyaiengine.DomainName("Wired domain", ".wired.com")
        dom2 = pyaiengine.DomainName("Other domain", ".serving-sys.com")

        dm = pyaiengine.DomainNameManager()
        dom1.callback = domain_callback
        dm.add_domain_name(dom1)
        dm.add_domain_name(dom2)

        self.stack.set_domain_name_manager(dm, "HTTP")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 1)

        data_cache = self.stack.get_cache_data("http", "host")
        self.assertEqual(len(data_cache), 2)

        # Shows the domain matched on HTTP.
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dm.show_matched_domains()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 3)

    def test14(self):
        """Verify cache release functionality."""

        self.stack.flows_timeout = 50000000 # No timeout :D

        self.inject("../pcapfiles/sslflow.pcap")

        tcp_flows = self.stack.tcp_flow_manager

        self.assertEqual(len(tcp_flows), 1)

        for flow in tcp_flows:
            self.assertNotEqual(flow.ssl_info, None)

        self.inject("../pcapfiles/accessgoogle.pcap")

        udp_flows = self.stack.udp_flow_manager

        self.assertEqual(len(udp_flows), 1)

        for flow in udp_flows:
            self.assertNotEqual(flow.dns_info, None)

        # release some of the caches
        self.stack.release_cache("ssl")

        for flow in tcp_flows:
            self.assertEqual(flow.ssl_info, None)

        # release all the caches
        self.stack.release_caches()

        for flow in tcp_flows:
            self.assertEqual(flow.ssl_info, None)
            self.assertEqual(flow.http_info, None)

        for flow in udp_flows:
            self.assertEqual(flow.dns_info, None)

    def test15(self):
        """Attach a database to the engine and test timeouts on udp flows."""

        adaptor = DatabaseTestAdaptor()

        self.stack.link_layer_tag = "vlan"
        self.stack.set_udp_database_adaptor(adaptor, 16)

        self.stack.flows_timeout = 1

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(adaptor.total_inserts, 1)
        self.assertEqual(adaptor.total_updates, 1)
        self.assertEqual(adaptor.total_removes, 1)
        self.assertEqual(self.stack.flows_timeout, 1)

    def test16(self):
        """Verify that ban domains dont take memory."""

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTP", False)

        self.inject("../pcapfiles/two_http_flows_noending.pcap", pcapfilter="tcp")

        self.assertEqual(dom.matchs, 1)

        tcp_flows = self.stack.tcp_flow_manager

        self.assertEqual(len(tcp_flows), 2)

        # Only the first flow is the banned
        flow = [flow for flow in tcp_flows][0]

        info = flow.http_info
        self.assertEqual(info.host_name, "")
        self.assertEqual(info.user_agent, "")
        self.assertEqual(info.uri, "")

    def test17(self):
        """Verify the ban functionality on the fly with a callback."""

        def domain_callback(flow):
            self.called_callback += 1

            info = flow.http_info
            url = info.uri

            # Some URI analsys on the first request could be done here
            if url == "/css/global.css?v=20121120a":
                info.banned = True

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTP")

        # the dm is plugged to the HTTPProtocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"HTTP", lines[0])

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 1)

        ft = self.stack.tcp_flow_manager

        self.assertEqual(len(ft), 2)

        items = [flow for flow in self.stack.tcp_flow_manager]

        # Only the first flow is the banned and released
        flow = [flow for flow in self.stack.tcp_flow_manager][0]
        
        info = flow.http_info
        self.assertNotEqual(info, None)
        self.assertEqual(info.uri, "")
        self.assertEqual(info.user_agent, "")
        self.assertEqual(info.host_name, "")

        self.stack.release_caches()

        self.stack.set_domain_name_manager(pyaiengine.DomainNameManager(), "HTTP")

        # The dm is plugged to the HTTPProtocol
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            dman.show()
            file.seek(0)
            file.flush()
            lines = file.readlines()
            self.assertNotIn(b"HTTPProtocol", lines[0])

    def test18(self):
        """Verify the getCounters functionality."""

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        counters = self.stack.get_counters("ethernet")

        if sys.version_info.major > 2:
            self.assertEqual("packets" in counters, True)
            self.assertEqual("bytes" in counters, True)
        else:
            self.assertEqual(counters.has_key("packets"), True)
            self.assertEqual(counters.has_key("bytes"), True)

        self.assertEqual(counters["bytes"], 910064)

        counters = self.stack.get_counters("tCp")

        self.assertEqual(counters["bytes"], 879940)
        self.assertEqual(counters["packets"], 886)
        self.assertEqual(counters["syns"], 2)
        self.assertEqual(counters["synacks"], 2)
        self.assertEqual(counters["acks"], 882)
        self.assertEqual(counters["rsts"], 0)
        self.assertEqual(counters["fins"], 0)

        counters = self.stack.get_counters("UnknownProtocol")
        self.assertEqual(len(counters), 0)

    def test19(self):
        """Verify SMTP traffic with domain callback."""

        self.from_correct = False
        def domain_callback(flow):
            smtp = flow.smtp_info
            if smtp:
                if str(smtp.mail_from) == "gurpartap@patriots.in":
                    self.from_correct = True
                _ = smtp.mail_to
            self.called_callback += 1

        dom = pyaiengine.DomainName("Some domain", ".patriots.in")
        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "smtp")

        oldstack = None

        with pyaiengine.PacketDispatcher("../pcapfiles/smtp.pcap") as pdis:
            pdis.stack = self.stack
            pdis.run()
            oldstack = pdis.stack

        self.assertEqual(oldstack, self.stack)

        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)
        self.assertEqual(self.from_correct, True)
        self.assertEqual(len(self.stack.get_cache_data("smtp", "from")), 1)

    def test20(self):
        """Test the chains of regex with RegexManagers."""

        rlist = [pyaiengine.Regex("expression %d" % x, "some regex %d" % x) for x in range(0, 5)]

        rmbase = pyaiengine.RegexManager(rlist)
        rman1 = pyaiengine.RegexManager()
        rman2 = pyaiengine.RegexManager()
        rman3 = pyaiengine.RegexManager()

        reg1 = pyaiengine.Regex("smtp1", "^AUTH LOGIN")
        reg1.next_regex_manager = rman1
        rmbase.add_regex(reg1)

        reg2 = pyaiengine.Regex("smtp2", "^NO MATCHS")
        reg3 = pyaiengine.Regex("smtp3", "^MAIL FROM")

        rman1.add_regex(reg2)
        rman1.add_regex(reg3)
        reg3.next_regex_manager = rman2

        reg4 = pyaiengine.Regex("smtp4", "^NO MATCHS")
        reg5 = pyaiengine.Regex("smtp5", "^DATA")

        rman2.add_regex(reg4)
        rman2.add_regex(reg5)
        reg5.next_regex_manager = rman3

        reg6 = pyaiengine.Regex("smtp6", "^QUIT")
        rman3.add_regex(reg6)

        self.stack.tcp_regex_manager = rmbase
        self.stack.mode = "nids"

        self.inject("../pcapfiles/smtp.pcap")

        for reg in rlist:
            self.assertEqual(reg.matchs, 0)

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(reg4.matchs, 0)
        self.assertEqual(reg5.matchs, 1)
        self.assertEqual(reg6.matchs, 1)

    def test21(self):
        """Tests the parameters of the callbacks."""

        def callback1(flow):
            # pylint: disable=unused-argument
            pass

        def callback2(flow, other):
            # pylint: disable=unused-argument
            pass

        def callback3(_):
            pass

        other_ref = callback3

        self.assertEqual(sys.getrefcount(callback1), 2)
        self.assertEqual(sys.getrefcount(callback2), 2)
        self.assertEqual(sys.getrefcount(callback3), 3)

        reg = pyaiengine.Regex("netbios", "CACACACA")
        try:
            reg.callback = None
            self.assertTrue(False)
        except:
            self.assertTrue(True)

        self.assertEqual(sys.getrefcount(callback1), 2)
        self.assertEqual(sys.getrefcount(callback2), 2)
        self.assertEqual(sys.getrefcount(callback3), 3)

        try:
            reg.callback = callback2
            self.assertTrue(False)
        except:
            self.assertTrue(True)

        self.assertEqual(sys.getrefcount(callback1), 2)
        self.assertEqual(sys.getrefcount(callback2), 2)
        self.assertEqual(sys.getrefcount(callback3), 3)

        try:
            reg.callback = callback1
            self.assertTrue(True)
        except:
            self.assertTrue(False)

        self.assertEqual(sys.getrefcount(callback1), 3)
        self.assertEqual(sys.getrefcount(callback2), 2)
        self.assertEqual(sys.getrefcount(callback3), 3)

        try:
            reg.callback = callback3
            self.assertTrue(True)
        except:
            self.assertTrue(False)

        self.assertEqual(sys.getrefcount(callback1), 2)
        self.assertEqual(sys.getrefcount(callback2), 2)
        self.assertEqual(sys.getrefcount(callback3), 4)

        reg.callback = None
        self.assertEqual(sys.getrefcount(callback3), 3)

    def test22(self):
        """ Verify the functionality of the HTTPUriSets with the callbacks """

        self.uset = pyaiengine.HTTPUriSet()
        def domain_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        def uri_callback(flow):
            self.assertIsNotNone(flow)
            self.assertEqual(len(self.uset), 1)
            self.assertEqual(self.uset.lookups, 39)
            self.assertEqual(self.uset.lookups_in, 1)
            self.assertEqual(self.uset.lookups_out, 38)
            self.called_callback += 1

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.uset.add_uri("/images_blogs/gadgetlab/2013/08/AP090714043057-60x60.jpg")
        self.uset.callback = uri_callback

        dom.http_uri_set = self.uset

        self.assertEqual(self.uset.callback, uri_callback)
        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(dom.http_uri_set, self.uset)
        self.assertEqual(len(self.uset), 1)
        self.assertEqual(self.uset.lookups, 39)
        self.assertEqual(self.uset.lookups_in, 1)
        self.assertEqual(self.uset.lookups_out, 38)

        self.assertEqual(self.called_callback, 2)

    def test23(self):
        """ Verify the functionality of the HTTPUriSets with the callbacks """

        self.uset = pyaiengine.HTTPUriSet()
        def domain_callback(flow):
            self.called_callback += 1

        def uri_callback(flow):
            self.assertEqual(len(self.uset), 1)
            self.assertEqual(self.uset.lookups, 4)
            self.assertEqual(self.uset.lookups_in, 1)
            self.assertEqual(self.uset.lookups_out, 3)
            self.called_callback += 1

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        # This uri is the thrid of the wired.com flow
        self.uset.add_uri("/js/scrolldock/scrolldock.css?v=20121120a")
        self.uset.callback = uri_callback

        dom.http_uri_set = self.uset

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(len(self.uset), 1)
        self.assertEqual(self.uset.lookups, 39)
        self.assertEqual(self.uset.lookups_in, 1)
        self.assertEqual(self.uset.lookups_out, 38)
        self.assertEqual(self.called_callback, 2)

    def test24(self):
        """ Verify the property of the PacketDispatcher.stack """

        pdis_original = pyaiengine.PacketDispatcher()
        pdis = pyaiengine.PacketDispatcher()

        self.assertEqual(pdis.stack, None)

        pdis_original.stack = None

        self.assertEqual(pdis_original.stack, None)

    def test25(self):
        """ Verify the functionality of the SSDP Protocol """

        def callback_ssdp(flow):
            self.assertEqual(flow.ssdp_info.uri, "*")
            self.assertEqual(flow.ssdp_info.host_name, "239.255.255.250:1900")
            self.called_callback += 1

        dom = pyaiengine.DomainName("All", "*", callback_ssdp)

        self.assertEqual(sys.getrefcount(dom), 2)

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)
        self.assertEqual(sys.getrefcount(dom), 3)

        # Remove and add again, just verify the ref count
        dman.remove_domain_name("All")
        self.assertEqual(sys.getrefcount(dom), 2)
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "ssdp")

        self.inject("../pcapfiles/ssdp_flow.pcap")

        self.assertEqual(self.called_callback, 1)

        data_cache = self.stack.get_cache_data("ssdp", "host")
        cache1 = {"239.255.255.250:1900": 1}
        self.assertDictEqual(data_cache, cache1)

    def test26(self):
        """Verify the functionality of the SSDP Protocol and remove the memory of that protocol."""

        self.stack.decrease_allocated_memory("ssdp", 10000)

        self.inject("../pcapfiles/ssdp_flow.pcap")

        fu = self.stack.udp_flow_manager
        for flow in fu:
            self.assertEqual(flow.ssdp_info, None)

    def test27(self):
        """Verify the functionality of the RegexManager on the HTTP Protocol for analise
            inside the l7 payload of HTTP."""

        def callback_domain(_):
            self.called_callback += 1
            pass

        def callback_regex(flow):
            self.called_callback += 1
            self.assertEqual(flow.packets, 11)
            self.assertEqual(flow.packets_layer7, 4)

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        reg1 = pyaiengine.Regex("Regex for analysing the content of HTTP", \
             b"^\\x1f\\x8b\\x08\\x00\\x00\\x00\\x00.*$")
        reg2 = pyaiengine.Regex("Regex for analysing the content of HTTP", \
            b"^.{3}\\xcd\\x9c\\xc0\\x0a\\x34.*$")
        reg3 = pyaiengine.Regex("Regex for analysing the content of HTTP", \
            b"^.*\\x44\\x75\\x57\\x0c\\x22\\x7b\\xa7\\x6d$")

        reg2.next_regex = reg3
        reg1.next_regex = reg2
        reg3.callback = callback_regex

        rman = pyaiengine.RegexManager("One manager", [reg1])

        # So the flows from wired.com will be analise the regexmanager attached
        dom.regex_manager = rman

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(dom.matchs, 1)

    def test28(self):
        """Verify the correctness of the HTTP Protocol."""

        # The filter tcp and port 55354 will filter just one HTTP flow
        # that contains exactly 39 requests and 38 responses
        self.inject("../pcapfiles/two_http_flows_noending.pcap", pcapfilter="tcp and port 55354")

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["requests"], 39)
        self.assertEqual(counters["responses"], 38)

    def test29(self):
        """Verify the correctness of the HTTP Protocol."""

        # The filter tcp and port 49503 will filter just one HTTP flow
        # that contains exactly 39 requests and 38 responses
        self.inject("../pcapfiles/two_http_flows_noending.pcap", pcapfilter="tcp and port 49503")

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["requests"], 3)
        self.assertEqual(counters["responses"], 3)

    def test30(self):
        """Verify the functionality of the Evidence manager."""

        def domain_callback(flow):
            self.called_callback += 1
            flow.evidence = True

        dom = pyaiengine.DomainName("Wired domain", ".wired.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTp")

        with pyaiengine.PacketDispatcher("../pcapfiles/two_http_flows_noending.pcap") as pdis:
            pdis.evidences = True
            pdis.stack = self.stack
            pdis.run()

        self.assertEqual(self.called_callback, 1)
        self.assertEqual(dom.matchs, 1)

        files = glob.glob("evidences.*.pcap")
        os.remove(files[0])

    def test31(self):
        """Verify the functionality of the RegexManager on the IPSets."""

        def regex_callback(flow):
            reg = flow.regex
            iset = flow.ip_set
            self.assertEqual(flow.dst_ip, "95.100.96.10")
            self.assertEqual(reg.name, "generic http")
            self.assertEqual(iset.name, "Generic set")
            self.called_callback += 1

        def ipset_callback(flow):
            reg = flow.regex
            iset = flow.ip_set
            self.assertNotEqual(iset, None)
            self.assertEqual(iset.name, "Generic set")
            self.assertEqual(reg, None)
            self.called_callback += 1

        rman = pyaiengine.RegexManager()
        ipset = pyaiengine.IPSet("Generic set", ["95.100.96.10"])
        ipset.regex_manager = rman
        ipset.callback = ipset_callback
        iman = pyaiengine.IPSetManager()

        iman.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = iman

        reg = pyaiengine.Regex("generic http", "^GET.*HTTP")
        reg.callback = regex_callback
        rman.add_regex(reg)

        self.stack.mode = "nids"

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(ipset.lookups_in, 1)
        self.assertEqual(reg.matchs, 1)

        # Verify the output of the IPSet
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            # The regex should be shown
            ipset.show()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 2)

        iman.reset()

        self.assertEqual(ipset.lookups_in, 0)
        self.assertEqual(ipset.lookups_out, 0)

    def test32(self):
        """ Verify the functionality of the RegexManager on the IPSets """

        def regex_callback(flow):
            reg = flow.regex
            ipset = flow.ip_set
            self.assertEqual(flow.dst_ip, "95.100.96.10")
            self.assertEqual(reg.name, "generic http")
            self.assertEqual(ipset.name, "Generic set")
            self.called_callback += 1

        def ipset_callback(flow):
            reg = flow.regex
            ipset = flow.ip_set
            self.assertNotEqual(ipset, None)
            self.assertEqual(ipset.name, "Generic set")
            self.assertEqual(reg, None)
            self.called_callback += 1

        rman = pyaiengine.RegexManager()
        ipset = pyaiengine.IPSet("Generic set")
        ipset.add_ip_address("95.100.96.10")
        ipset.regex_manager = rman
        ipset.callback = ipset_callback
        iman = pyaiengine.IPSetManager()

        iman.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = iman

        reg = pyaiengine.Regex("generic http", "^GET.*HTTP")
        reg.callback = regex_callback
        rman.add_regex(reg)

        self.stack.mode = "nids"

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(ipset.lookups_in, 1)
        self.assertEqual(reg.matchs, 1)

    def test33(self):
        """ Verify the clean of domains on the domain name manager """
        dman = pyaiengine.DomainNameManager("One domain manager", [
            pyaiengine.DomainName("Wired domain", ".wired.com"),
            pyaiengine.DomainName("Wired domain", ".photos.wired.com"),
            pyaiengine.DomainName("Wired domain", ".aaa.wired.com"),
            pyaiengine.DomainName("Wired domain", ".max.wired.com"),
            pyaiengine.DomainName("domain1", ".paco.com"),
            pyaiengine.DomainName("domain2", ".cisco.com")])

        self.assertEqual(len(dman), 6)

        dman.remove_domain_name("domain1")
        self.assertEqual(len(dman), 5)

        dman.remove_domain_name("Wired domain")
        self.assertEqual(len(dman), 1)

    def test34(self):
        """Verify the functionality write on the databaseAdaptor when an
        important event happen on UDP."""

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("my regex", b"^HTTP.*$")

        # Write the packet
        reg.write_packet = True

        rman.add_regex(reg)

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.stack.udp_regex_manager = rman

        self.stack.mode = "nids"

        self.inject("../pcapfiles/ssdp_flow.pcap")

        data = json.loads(adaptor.lastdata)
        if "matchs" in data:
            self.assertEqual(data["matchs"], "my regex")
        self.assertEqual(reg.matchs, 1)

        # The packet is write on the packet field of the json
        packet = data["l7_payload"]
        cad = "".join(str(chr(x)) for x in packet)
        self.assertEqual(cad.startswith("HTTP"), True)

    def test35(self):
        """ Verify the coap protocol functionality """

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/ipv4_coap.pcap")

        data = json.loads(adaptor.lastdata)
        if "info" in data:
            self.assertEqual(data["info"]["uri"], "/1/1/768/core.power")

        # Release the cache for coap
        self.assertEqual(len(self.stack.udp_flow_manager), 1)

        for flow in self.stack.udp_flow_manager:
            self.assertNotEqual(flow.coap_info, None)

        # release  the cache
        self.stack.release_cache("CoAP")

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.coap_info, None)

        # release all the caches
        self.stack.release_caches()

    def test36(self):
        """ Verify the mqtt protocol functionality """

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/ipv4_mqtt.pcap")

        data = json.loads(adaptor.lastdata)
        if "info" in data:
            self.assertEqual(data["info"]["operation"], 4)
            self.assertEqual(data["info"]["total_server"], 9)
            self.assertEqual(data["info"]["total_client"], 8)

        # print(json.dumps(d,sort_keys=True,indent=4, separators=(",", ": ")))

        # Release the cache for mqtt
        self.assertEqual(len(self.stack.tcp_flow_manager), 1)

        for flow in self.stack.tcp_flow_manager:
            self.assertNotEqual(flow.mqtt_info, None)
            self.assertEqual(flow.coap_info, None)
            self.assertEqual(flow.http_info, None)
            self.assertEqual(flow.dns_info, None)
            self.assertEqual(flow.ssl_info, None)

        # release  the cache
        self.stack.release_cache("MQTT")

        for flow in self.stack.tcp_flow_manager:
            self.assertEqual(flow.mqtt_info, None)
            self.assertEqual(flow.coap_info, None)
            self.assertEqual(flow.http_info, None)
            self.assertEqual(flow.dns_info, None)
            self.assertEqual(flow.ssl_info, None)

        # release all the caches
        self.stack.release_caches()

    def test37(self):
        """ Verify the coap protocol functionality with domains matched """

        def domain_callback(flow):
            self.called_callback += 1
            self.assertNotEqual(flow.coap_info, None)
            self.assertEqual(flow.coap_info.host_name, "localhost")
            self.assertEqual(flow.coap_info.uri, \
                "/somepath/really/maliciousuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuua/time")
            # the label is the concatenation of the host and the uri
            flow.label = flow.coap_info.host_name + flow.coap_info.uri
            self.assertEqual(flow.coap_info.matched_domain_name, dom)

        dom = pyaiengine.DomainName("Localhost domain", "localhost")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "CoAP")

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/ipv4_coap_big_uri.pcap")

        data = json.loads(adaptor.lastdata)
        # print(json.dumps(data,sort_keys=True,indent=4, separators=(",", ": ")))
        if "coap" in data:
            self.assertEqual(data["coap"]["uri"], \
                "/somepath/really/maliciousuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuua/time")
        else:
            self.assertTrue(False)

        self.assertEqual(data["label"], \
            "localhost/somepath/really/maliciousuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuua/time")
        self.assertEqual(self.called_callback, 1)
        self.assertEqual(dom.matchs, 1)

    def test38(self):
        """Test the modbus protocol."""

        self.inject("../pcapfiles/modbus_five_flows.pcap")

        counters = self.stack.get_counters("ModbUS")
        self.assertEqual(counters["write single coil"], 4)
        self.assertEqual(counters["read coils"], 6)

    def test39(self):
        """Verify the release cache with netbios object attached."""
        self.stack.link_layer_tag = "vlan"

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        flow = [flow for flow in self.stack.udp_flow_manager][0]

        self.assertIsNotNone(flow)
        self.assertIsNotNone(flow.netbios_info)
        self.assertIsNotNone(flow.netbios_info.name)
        self.assertEqual(flow.netbios_info.name, "BLUMGROUP")

        self.stack.release_cache("netbios")

        flow = [flow for flow in self.stack.udp_flow_manager][0]

        self.assertIsNotNone(flow)
        self.assertIsNone(flow.netbios_info)

    def test40(self):
        """Verify that callbacks with None do not break things."""

        def domain_callback(_):
            self.called_callback += 1

        dom = pyaiengine.DomainName("Google Drive Cert", ".drive.google.com")
        dom.callback = None

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "SSL")

        self.inject("../pcapfiles/sslflow.pcap")

        # First time nothing happens
        self.assertEqual(len(dman), 1)
        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 0)

        # flush the flows from memory
        self.stack.tcp_flow_manager.flush()

        # reinject the flows with the callback set
        dom.callback = domain_callback

        self.inject("../pcapfiles/sslflow.pcap")

        # Second time callback is executed
        self.assertEqual(len(dman), 1)
        self.assertEqual(dom.matchs, 2)
        self.assertEqual(self.called_callback, 1)

        # flush the flows from memory again
        self.stack.tcp_flow_manager.flush()

        # reinject the flows with the callback set to None
        dom.callback = None

        self.inject("../pcapfiles/sslflow.pcap")

        # Second time callback is not executed
        self.assertEqual(len(dman), 1)
        self.assertEqual(dom.matchs, 3)
        self.assertEqual(self.called_callback, 1)

    def test41(self):
        """Create a regex for netbios and add and remove from a RegexManager."""
        self.stack.link_layer_tag = "vlan"

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("netbios", "CACACACA")
        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman

        self.stack.mode = "nids"

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(len(rman), 1)
        self.assertEqual(self.stack.udp_regex_manager, rman)
        self.assertEqual(self.stack.link_layer_tag, "vlan")

        rman.remove_regex(reg)

        self.assertEqual(len(rman), 0)

        self.stack.udp_flow_manager.flush()

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg.matchs, 1)

    def test42(self):
        """Create a regex for netbios with callback and a RegexManager with callback."""

        def callback_rm(flow):
            self.called_callback += 1
            self.assertEqual(flow.regex.matchs, 1)
            self.assertEqual(flow.regex.name, "netbios")

        def callback_nb(flow):
            self.assertIsNotNone(flow)
            self.fail("shouldn't happen")

        self.stack.link_layer_tag = "vlan"

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("netbios", "CACACACA")

        reg.callback = callback_nb

        # The regex manager sets a callback so the regexs will not call their own callbacks
        rman.callback = callback_rm

        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman

        self.stack.mode = "nids"

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(self.called_callback, 1)

    def test43(self):
        """"Create a complex detection on http traffic payload for exercise the code."""

        def callback_domain(flow):
            self.called_callback += 1
            self.assertIsNone(flow.regex_manager)
            self.assertEqual(flow.http_info.matched_domain_name.regex_manager, rman1)

        def callback_regex1(flow):
            self.assertEqual(flow.packets, 7)
            self.assertEqual(flow.packets_layer7, 3)
            self.assertIsNotNone(flow.regex_manager)
            self.assertEqual(flow.regex_manager.name, rman2.name)

        def callback_regex2(flow):
            self.assertEqual(flow.packets, 40)
            self.assertEqual(flow.packets_layer7, 20)
            self.assertEqual(flow.regex_manager.name, rman2.name)

        def callback_regex3(flow):
            self.called_callback += 1
            self.assertEqual(flow.packets, 90)
            self.assertEqual(flow.packets_layer7, 47)

        dom = pyaiengine.DomainName("Some domain", ".serving-sys.com")

        rman1 = pyaiengine.RegexManager("Im the first regexs")
        rman2 = pyaiengine.RegexManager("Im the second regexs")
        reg1 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^.*Ducky.*$")
        reg2 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^.*Ducky.*$")
        reg3 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^.*Photoshop.*$")

        reg2.next_regex = reg3
        self.assertEqual(sys.getrefcount(reg3), 3)

        reg1.next_regex_manager = rman2

        rman1.add_regex(reg1)
        rman2.add_regex(reg2)

        reg1.callback = callback_regex1
        reg2.callback = callback_regex2
        reg3.callback = callback_regex3

        # Attach the regexmanager to the domain
        dom.regex_manager = rman1

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(dom.matchs, 1)

        reg2.next_regex = None
        self.assertEqual(sys.getrefcount(reg3), 2)

    def test44(self):
        """Test the chains of regex with callbacks and regex on constructors."""

        def callback_regex_auth(_):
            self.called_callback += 1

        def callback_regex_from(_):
            self.called_callback += 1

        def callback_regex_data(_):
            self.called_callback += 1

        def callback_regex_quit(_):
            self.called_callback += 1

        # Example of link regexs with callbacks inside
        reg = pyaiengine.Regex("smtp1", b"^AUTH LOGIN.*$", callback_regex_auth, \
            pyaiengine.Regex("smtp2", b"^MAIL FROM.*$", callback_regex_from, \
            pyaiengine.Regex("smtp3", b"^DATA.*$", callback_regex_data, \
            pyaiengine.Regex("smtp4", b"^QUIT.*$", callback_regex_quit))))

        rman = pyaiengine.RegexManager([reg])

        self.stack.tcp_regex_manager = rman
        self.stack.mode = "nids"

        self.inject("../pcapfiles/smtp.pcap")

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(reg.next_regex.matchs, 1)
        self.assertEqual(reg.next_regex.next_regex.matchs, 1)
        self.assertEqual(reg.next_regex.next_regex.next_regex.matchs, 1)
        self.assertEqual(self.called_callback, 4)

    def test45(self):
        """Test use of regexs on the HTTP uri field."""

        def callback_uri(flow):
            inf = flow.http_info
            self.assertNotEqual(inf, None)
            self.assertEqual(inf.uri, "/textinputassistant/tia.png")
            self.assertEqual(inf.host_name, "www.google.com")
            self.assertEqual(inf.total_requests, 2)
            self.assertEqual(inf.total_responses, 1)
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("my uri regex", b"^.*tia.png$", callback_uri)
        dom = pyaiengine.DomainName("Gafas", ".google.com")

        # Attach the RegexManager to process all the Uris from google
        dom.http_uri_regex_manager = rman

        self.assertEqual(dom.http_uri_regex_manager, rman)

        dman.add_domain_name(dom)
        rman.add_regex(reg)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/accessgoogle.pcap")

        self.assertEqual(self.called_callback, 1)

    def test46(self):
        """Verify the functionality of dynamic memory with the SSDP Protocol."""

        self.stack.decrease_allocated_memory("ssdp", 10000)
        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/ssdp_flow.pcap")

        for flow in self.stack.udp_flow_manager:
            self.assertNotEqual(flow.ssdp_info, None)

    def test47(self):
        """Verify the functionality of dynamic memory with the HTTP Protocol."""

        self.stack.decrease_allocated_memory("HTTP", 10000)

        # enable the dynamic memory for just http
        self.stack.set_dynamic_allocated_memory("HTTP", True)

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        for flow in self.stack.tcp_flow_manager:
            self.assertNotEqual(flow.http_info, None)
            self.assertNotEqual(flow.http_info.host_name, None)
            self.assertNotEqual(flow.http_info.uri, None)
            self.assertNotEqual(flow.http_info.user_agent, None)

    def test48(self):
        """Complex detection on the HTTP Protocol."""

        def callback_1(flow):
            self.assertEqual(flow.packets_layer7, 2)

            # The first uri should match on this point
            self.assertNotEqual(flow.http_info, None)
            self.assertEqual(flow.http_info.uri, "/css/global.css?v=20121120a")
            self.called_callback += 1

        def callback_2(flow):
            self.assertEqual(flow.packets_layer7, 5)

            # The first uri should match on this point because didnt change
            self.assertNotEqual(flow.http_info, None)
            self.assertEqual(flow.http_info.uri, "/css/global.css?v=20121120a")
            self.called_callback += 1

        def callback_3(flow):
            self.assertEqual(flow.packets_layer7, 15)
            self.assertNotEqual(flow.http_info, None)
            self.assertEqual(flow.http_info.uri,
                             "/images_blogs/gadgetlab/2013/07/MG_9640edit-200x100.jpg")
            self.called_callback += 1

        def callback_4(flow):
            self.assertEqual(flow.packets_layer7, 16)
            self.assertNotEqual(flow.http_info, None)
            self.assertEqual(flow.http_info.uri,
                             "/images_blogs/gadgetlab/2013/07/MG_9640edit-200x100.jpg")
            self.called_callback += 1

        def callback_5(flow):
            self.assertEqual(flow.packets_layer7, 31)
            self.assertNotEqual(flow.http_info, None)
            self.assertEqual(flow.http_info.uri,
                             "/images_blogs/thisdayintech/2013/03/set.jpg")
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("matchs on 1 response", \
             b"^\\x1f\\x8b\\x08\\x00\\x00\\x00\\x00\\x00\\x00\\x00.*$", \
             callback_1)
        reg2 = pyaiengine.Regex("matchs on last response", \
            b"^.*\\x5a\\xf2\\x74\\x8f\\x39\\x4e\\x00\\x00$", callback_2)
        reg3 = pyaiengine.Regex("matchs on 4 response", \
            b"^\\xff\\xd8\\xff\\xe0\\x00\\x10\\x4a\\x46\\x49\\x46.*$", callback_3)
        reg4 = pyaiengine.Regex("matchs on 4 response", \
            b"^\\xf6\\xae\\x30\\x7a\\x1f\\x3c\\xea\\x7e.*$", callback_4)
        reg5 = pyaiengine.Regex("matchs on other response", \
            b"^\\xff\\xd8\\xff\\xe1\\x00\\x18\\x45\\x78\\x69.*$", callback_5)
        dom = pyaiengine.DomainName("No trusted domain", ".wired.com")

        dman.add_domain_name(dom)

        rman.add_regex(reg1)

        reg1.next_regex = reg2
        reg2.next_regex = reg3
        reg3.next_regex = reg4
        reg4.next_regex = reg5

        # Attach the RegexManager to process all the payloads from wired
        dom.regex_manager = rman

        # enable the dynamic memory for just http
        self.stack.set_dynamic_allocated_memory("HTTP", True)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 5)
        self.assertEqual(dom.matchs, 1)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(reg4.matchs, 1)
        self.assertEqual(reg5.matchs, 1)

    def test49(self):
        """Verify the ban domains on SSL traffic."""

        dom = pyaiengine.DomainName("Google Drive Cert", ".drive.google.com")
        dom.callback = None

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "ssl", False)

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(dom.matchs, 1)

        flow = [flow for flow in self.stack.tcp_flow_manager][0]

        cache = self.stack.get_cache_data("ssl", "issuer")
        self.assertNotEqual(flow, None)
        self.assertNotEqual(flow.ssl_info, None)
        self.assertEqual(flow.ssl_info.server_name, "")
        self.assertEqual(len(cache), 0)

        # Check also the upstream and downstream packets/bytes
        self.assertEqual(flow.upstream_packets, 42)
        self.assertEqual(flow.downstream_packets, 53)
        self.assertEqual(flow.packets, flow.upstream_packets + flow.downstream_packets)
        self.assertEqual(flow.upstream_bytes, 10997)
        self.assertEqual(flow.downstream_bytes, 30824)
        self.assertEqual(flow.bytes, flow.upstream_bytes + flow.downstream_bytes)

    def test50(self):
        """ Verify the order of the flow when iterate through the flow manager
            192.168.1.1:57077:6:54.230.87.203:443
            192.168.1.1:57080:6:54.230.87.203:443
            192.168.1.1:57079:6:54.230.87.203:443
            192.168.1.1:57078:6:54.230.87.203:443
        """

        source_port = 57077
        self.inject("../pcapfiles/amazon_4ssl_flows.pcap")

        self.assertEqual(len(self.stack.tcp_flow_manager), 4)

        items = [flow for flow in self.stack.tcp_flow_manager]

        self.assertEqual(items[0].src_port, 57077)
        self.assertEqual(items[1].src_port, 57080)
        self.assertEqual(items[2].src_port, 57079)
        self.assertEqual(items[3].src_port, 57078)

    def test51(self):
        """Verify the functionality of the IPRadixTrees."""

        def regex_callback(flow):
            reg = flow.regex
            ipset = flow.ip_set
            if sys.version_info.major > 2:
                self.assertRegex(flow.dst_ip, "(95.100.96.10|95.100.96.48)")
            else:
                self.assertRegexpMatches(flow.dst_ip, "(95.100.96.10|95.100.96.48)")
            self.assertEqual(reg.name, "generic http")
            self.assertEqual(ipset.name, "something")
            self.called_callback += 1

        def ipset_callback(flow):
            reg = flow.regex
            ipset = flow.ip_set
            self.assertNotEqual(ipset, None)
            self.assertEqual(ipset.name, "something")
            self.assertEqual(reg, None)
            self.called_callback += 1

        rman = pyaiengine.RegexManager()
        rtree = pyaiengine.IPRadixTree(["95.100.96.10/24", "192.172.12.1"])

        # Change the name of the radix tree
        self.assertEqual(rtree.name, "Generic IPRadixTree")
        rtree.name = "something"

        rtree.regex_manager = rman
        rtree.callback = ipset_callback
        im = pyaiengine.IPSetManager()

        self.assertEqual(rtree.callback, ipset_callback)

        self.assertEqual("Generic IPSetManager", im.name)
        im.name = "buuu"
        self.assertEqual("buuu", im.name)

        im.add_ip_set(rtree)
        self.stack.tcp_ip_set_manager = im

        reg = pyaiengine.Regex("generic http", "^GET.*HTTP")
        reg.callback = regex_callback
        rman.add_regex(reg)

        self.stack.mode = "nids"

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback, 4)
        self.assertEqual(rtree.lookups_in, 2)
        self.assertEqual(reg.matchs, 2)

        # Verify the output of the IPRadixTree
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            rtree.show()
            file.seek(0)
            lines = file.readlines()
            total_lines = len(lines)
            self.assertEqual(total_lines, 3)
            self.assertIn(b"95.100.96.0/24", lines[1])
            if (platform.machine().upper() == "ARM64"):
                self.assertIn(b"...", lines[2])
            else:
                self.assertIn(b"192.172.12.1", lines[2])

    def test52(self):
        """Verify that we have two different records on the adaptors with smtp-starttls."""

        adaptor = DatabaseTestAdaptor()

        # with 16 we generate two records
        self.stack.set_tcp_database_adaptor(adaptor, 16)

        self.inject("../pcapfiles/smtp_starttls.pcap")

        self.assertEqual(len(adaptor.all_data), 2)

        smtp_r = json.loads(adaptor.all_data[1])
        ssl_r = json.loads(adaptor.all_data[2])

        self.assertEqual(smtp_r["layer7"], "SMTP")
        self.assertEqual(ssl_r["layer7"], "SSL")
        self.assertEqual(ssl_r["ssl"]["issuer"], "Google Internet Authority G2")

        self.assertEqual(smtp_r["smtp"]["tls"], True)
        self.assertEqual(smtp_r["ip"]["src"], ssl_r["ip"]["src"])
        self.assertEqual(smtp_r["ip"]["dst"], ssl_r["ip"]["dst"])
        self.assertEqual(smtp_r["port"]["src"], ssl_r["port"]["src"])
        self.assertEqual(smtp_r["port"]["dst"], ssl_r["port"]["dst"])

    def test53(self):
        """Verify that we have two different records on the adaptors with imap-starttls."""

        adaptor = DatabaseTestAdaptor()

        # with 16 we generate two records
        self.stack.set_tcp_database_adaptor(adaptor, 16)

        self.inject("../pcapfiles/imap_starttls.pcap")

        self.assertEqual(len(adaptor.all_data), 2)

        imap_r = json.loads(adaptor.all_data[1])
        ssl_r = json.loads(adaptor.all_data[2])

        self.assertEqual(imap_r["layer7"], "IMAP")
        self.assertEqual(ssl_r["layer7"], "SSL")

        self.assertEqual(imap_r["imap"]["tls"], True)
        self.assertEqual(imap_r["ip"]["src"], ssl_r["ip"]["src"])
        self.assertEqual(imap_r["ip"]["dst"], ssl_r["ip"]["dst"])
        self.assertEqual(imap_r["port"]["src"], ssl_r["port"]["src"])
        self.assertEqual(imap_r["port"]["dst"], ssl_r["port"]["dst"])

    def test54(self):
        """ verify that we have two different records on the adaptors with pop-starttls """

        adaptor = DatabaseTestAdaptor()

        # with 16 we generate two records
        self.stack.set_tcp_database_adaptor(adaptor, 16)

        self.inject("../pcapfiles/pop3_starttls.pcap")

        self.assertEqual(len(adaptor.all_data), 2)

        pop_r = json.loads(adaptor.all_data[1])
        ssl_r = json.loads(adaptor.all_data[2])

        self.assertEqual(pop_r["layer7"], "POP")
        self.assertEqual(ssl_r["layer7"], "SSL")

        self.assertEqual(pop_r["pop"]["tls"], True)
        self.assertEqual(pop_r["ip"]["src"], ssl_r["ip"]["src"])
        self.assertEqual(pop_r["ip"]["dst"], ssl_r["ip"]["dst"])
        self.assertEqual(pop_r["port"]["src"], ssl_r["port"]["src"])
        self.assertEqual(pop_r["port"]["dst"], ssl_r["port"]["dst"])

    def test55(self):

        def callback_domain(flow):
            info = flow.http_info
            self.assertEqual(flow.packets_layer7, 1)
            self.assertEqual(info.uri, "/index.htm?v=5&eh=&ts=0&u2=lpdDC5KtfXqwOCkfKJ0O")
            self.assertEqual(flow.regex, None)
            self.called_callback += 1

        def callback(flow):
            info = flow.http_info
            self.assertEqual(flow.packets_layer7, 2)
            self.assertEqual(info.uri, "/index.htm?v=5&eh=&ts=0&u2=lpdDC5KtfXqwOCkfKJ0O")
            self.assertEqual(flow.regex.name, reg.name)
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("matchs on 1 response", b"^.*PNG.*(?!.*IHDR).*$", callback)
        dom = pyaiengine.DomainName("No trusted domain", ".ru", callback_domain)

        dman.add_domain_name(dom)

        rman.add_regex(reg)

        # Attach the RegexManager to process all the payloads from wired
        dom.regex_manager = rman

        # enable the dynamic memory for just http
        self.stack.set_dynamic_allocated_memory("HTTP", True)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/http_flow.pcap")

        self.assertEqual(self.called_callback, 2)

    def test56(self):

        def callback_domain(flow):
            s = flow.smtp_info
            # Spammy address
            self.assertEqual(s.mail_from, "IVepijy@UTkSgBvIxlGQiKRIhmDTUxnmrOwzE.gov")
            self.called_callback += 1

        def callback_anomaly(flow):
            s = flow.smtp_info
            self.assertEqual(s.mail_from, "IVepijy@UTkSgBvIxlGQiKRIhmDTUxnmrOwzE.gov")
            self.assertGreater(len(flow.payload), 512)
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        dom = pyaiengine.DomainName("No trusted domain", ".gov", callback_domain)

        dman.add_domain_name(dom)

        self.stack.set_dynamic_allocated_memory("SMTP", True)

        # Set the anomaly manager callback
        self.stack.set_anomaly_callback(callback_anomaly, "smtp")
        self.stack.set_anomaly_callback(callback_anomaly, "SMTP")
        self.stack.set_anomaly_callback(callback_anomaly, "smtpprotocol")

        self.stack.set_domain_name_manager(dman, "SMTP")

        self.inject("../pcapfiles/smtp_flow.pcap")

        self.assertEqual(self.called_callback, 2)

        # Verify the output of the anomaly that have been set
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show(5)

    def test57(self):
        """Verify the counters and json output of the DCERPC component."""

        adaptor = DatabaseTestAdaptor()

        # with 16 we generate two records
        self.stack.set_tcp_database_adaptor(adaptor, 16)

        self.inject("../pcapfiles/dcerpc_traffic.pcapng")

        data = json.loads(adaptor.all_data[1])
        counters = self.stack.get_counters("dcerpc")

        self.assertEqual(data["layer7"], "DCERPC")
        self.assertEqual("uuid" in data["dcerpc"], True)

        self.assertEqual(counters["binds"], 10)
        self.assertEqual(counters["bind acks"], 10)

        for flow in self.stack.tcp_flow_manager:
            _ = flow.dcerpc_info.uuid

        data_cache = self.stack.get_cache_data("dcerpc", "uuid")
        self.assertEqual(len(data_cache), 3)

    def test58(self):
        """ disable DCERPC protocol and check values """

        self.stack.set_dynamic_allocated_memory(True)
        self.stack.disable_protocol("dcerpc")

        self.inject("../pcapfiles/dcerpc_traffic.pcapng")

        counters = self.stack.get_counters("dcerpc")

        self.assertEqual(counters["binds"], 0)
        self.assertEqual(counters["bind acks"], 0)

    def test59(self):
        """ disable DNS and HTTP protocol and check values """

        self.stack.disable_protocol("dns")
        self.stack.disable_protocol("HTTP")

        self.inject("../pcapfiles/accessgoogle.pcap")

        counters1 = {"L7 bytes": 0, "heads": 0, "responses": 0, "puts": 0, "packets": 0, "bytes": 0,
                     "connects": 0, "options": 0, "posts": 0, "banned hosts": 0, "others": 0,
                     "requests": 0, "gets": 0, "traces": 0, "allow hosts": 0, "deletes": 0}
        counters = self.stack.get_counters("http")

        self.assertDictEqual(counters, counters1)

        counters2 = {"type SRV": 0, "type AAAA": 0, "type SSHFP": 0, "type LOC": 0, "type PTR": 0,
                     "type NS": 0, "type A": 0, "type MX": 0, "type ANY": 0, "allow queries": 0,
                     "type IXFR": 0, "type DNSKEY": 0, "type others": 0, "queries": 0,
                     "type CNAME": 0, "responses": 0, "type SOA": 0, "banned queries": 0,
                     "type DS": 0, "type TXT": 0, "packets": 0, "bytes": 0, "type HTTPS": 0,
                     "type TLSA": 0, "rcode noerror": 0, "rcode formerr": 0, "rcode servfail": 0,
                     "rcode nxdomain": 0, "rcode notimp": 0, "rcode refused": 0, "rcode yxdomain": 0,
                     "rcode xrrset": 0, "rcode notauth": 0, "rcode notzone": 0, "rcode others": 0}

        counters = self.stack.get_counters("DNS")

        self.assertDictEqual(counters, counters2)

        counters3 = {"bytes": 1826, "packets": 4}
        counters = self.stack.get_counters("tcpgeneric") # take the HTTP traffic

        self.assertDictEqual(counters, counters3)

        # flush the flow tables
        self.stack.tcp_flow_manager.flush()
        self.stack.udp_flow_manager.flush()

        self.stack.enable_protocol("DNS")
        self.stack.enable_protocol("HTTP")

        self.inject("../pcapfiles/accessgoogle.pcap")

        counters = self.stack.get_counters("HtTp")

        counters4 = {"L7 bytes": 218, "heads": 0, "responses": 2, "puts": 0, "packets": 4,
                     "bytes": 1826, "connects": 0, "options": 0, "posts": 0, "banned hosts": 0,
                     "others": 0, "requests": 2, "gets": 2, "traces": 0, "allow hosts": 2,
                     "deletes": 0}

        self.assertDictEqual(counters, counters4)

        counters5 = {"type MX": 0, "type DS": 0, "type SOA": 0, "type CNAME": 0, "responses": 2,
                     "type SRV": 0, "type TXT": 0, "type ANY": 0, "type others": 0, "type SSHFP": 0,
                     "type LOC": 0, "type DNSKEY": 0, "type IXFR": 0, "type AAAA": 1, "type NS": 0,
                     "queries": 2, "allow queries": 2, "banned queries": 0, "type TLSA": 0,
                     "type PTR": 0, "type A": 1, "bytes": 252, "packets": 4, "type HTTPS": 0,
                     "rcode noerror": 2, "rcode formerr": 0, "rcode servfail": 0, "rcode nxdomain": 0,
                     "rcode notimp": 0, "rcode refused": 0, "rcode yxdomain": 0, "rcode xrrset": 0,
                     "rcode notauth": 0, "rcode notzone": 0, "rcode others": 0}

        counters = self.stack.get_counters("DNS")

        self.assertDictEqual(counters, counters5)

        counters = self.stack.get_counters("tcpgeneric") # Should be the same as before

        self.assertDictEqual(counters, counters3)

    def test60(self):
        """Operate with pop traffic."""

        def pop_callback(flow):
            self.assertEqual(flow.pop_info.user_name, "plod")

            # Check some TCP Flags
            tcp = flow.tcp_info
            self.assertIsNotNone(tcp)
            self.assertEqual(tcp.syns, 1)
            self.assertEqual(tcp.synacks, 1)
            self.assertEqual(tcp.synacks, 1)
            self.assertEqual(tcp.acks, 3)
            self.assertEqual(tcp.fins, 0)
            self.assertEqual(tcp.rsts, 0)
            self.assertEqual(tcp.pushs, 2)
            self.assertEqual(tcp.state, "ESTABLISHED")

            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        dom = pyaiengine.DomainName("No trusted domain", "*", pop_callback)

        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "pop")
        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/pop_flow.pcap")

        counters = self.stack.get_counters("pop")

        self.assertEqual(counters["commands"], 15)
        self.assertEqual(counters["responses"], 16)
        self.assertEqual(self.called_callback, 1)

        data_cache = self.stack.get_cache_data("pop", "user")
        self.assertIn("plod", data_cache)

    def test61(self):
        """Test case for IMAP traffic."""

        def imap_callback(flow):
            self.assertEqual(flow.imap_info.user_name, "samir")
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        dom = pyaiengine.DomainName("No trusted domain", "*", imap_callback)

        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "imap")
        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/imap_flow.pcap")

        counters = self.stack.get_counters("imap")

        self.assertEqual(counters["commands"], 6)
        self.assertEqual(counters["responses"], 12)
        self.assertEqual(self.called_callback, 1)

        data_cache = self.stack.get_cache_data("imap", "user")

        self.assertIn("samir", data_cache)

    def test62(self):
        """Test case for SMB traffic."""

        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/smb_flow.pcap")

        counters = self.stack.get_counters("smb")

        self.assertEqual(counters["create files"], 8)

        data_cache = self.stack.get_cache_data("smb", "name")
        self.assertEqual(len(data_cache), 0)

        data_cache = self.stack.get_cache_data("smb", "filename")
        self.assertEqual(len(data_cache), 2)

        for flow in self.stack.tcp_flow_manager:
            filename = flow.smb_info.filename

        self.assertEqual(filename, "WP_SMBPlugin.pdf")

    def test63(self):
        """Test case for MQTT traffic."""

        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/ipv4_mqtt.pcap")

        for flow in self.stack.tcp_flow_manager:
            _ = flow.mqtt_info.topic

        counters = self.stack.get_counters("mqtt")

        self.assertEqual(counters["commands"], 8)

        data_cache = self.stack.get_cache_data("mqtt", "topic")
        self.assertEqual(len(data_cache), 1)

    def test64(self):
        """ Test case for matchs several Regexs on DCERPC """

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("reg1", b"^\\x05\\x00\\x00\\x83\\x10\\x00\\x00\\x00\\x6c.*$")
        reg2 = pyaiengine.Regex("reg2", b"^\\x05\\x00\\x0b\\x03.*$")
        reg3 = pyaiengine.Regex("reg3", b"^.*\\xde\\xed\\xfc\\x0c.*$")
        reg4 = pyaiengine.Regex("reg4", b"^\\xde\\xed\\xfc\\x0c.*$")

        rman.add_regex(reg1)
        rman.add_regex(reg2)
        rman.add_regex(reg3)
        rman.add_regex(reg4)

        self.stack.mode = "nids"
        self.stack.tcp_regex_manager = rman

        self.stack.disable_protocol("dcerpc")

        self.inject("../pcapfiles/dcerpc_traffic.pcapng")

        self.assertEqual(reg1.matchs, 3)
        self.assertEqual(reg2.matchs, 8)
        self.assertEqual(reg3.matchs, 0)
        self.assertEqual(reg4.matchs, 0)

        # Shows the matched regexs
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            rman.show_matched_regexs()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 3)

    def test65(self):
        """Verify the coap protocol functionality with domains and uri sets."""

        def domain_callback(_):
            self.called_callback += 1

        def uri_callback(_):
            self.assertEqual(len(uset), 1)
            self.assertEqual(uset.lookups, 1)
            self.assertEqual(uset.lookups_in, 1)
            self.assertEqual(uset.lookups_out, 0)
            self.called_callback += 1

        uset = pyaiengine.HTTPUriSet()
        uset.add_uri("/somepath/really/maliciousuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuua/time")
        uset.callback = uri_callback

        dom = pyaiengine.DomainName("Localhost domain", "localhost")
        dom.http_uri_set = uset

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "CoAP")

        self.inject("../pcapfiles/ipv4_coap_big_uri.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(dom.matchs, 1)

        data_cache = self.stack.get_cache_data("coap", "host")
        c1 = {"localhost": 1}
        self.assertDictEqual(data_cache, c1)

    def test66(self):
        """ Verify that a Regex that matchs with a URI could handle the link
            to another RegexManager that will have regex for the payload of the HTTP """

        def domain_callback(flow):
            self.called_callback += 1
            self.assertEqual(flow.packets_layer7, 1)
            """ The flow.regex_manager is null on this point and assign
                the flow.http_info.matched_domain_name.regex_manager after this call """
            self.assertIsNone(flow.regex_manager)

        def uri_callback(flow):
            self.called_callback += 1
            self.assertEqual(flow.packets_layer7, 1)
            self.assertIsNone(flow.regex_manager)

        def payload_callback(flow):
            self.called_callback += 1
            self.assertEqual(flow.packets_layer7, 2)
            self.assertEqual(flow.regex_manager.name, rman1.name)

        dom = pyaiengine.DomainName("All HTTP", "*", domain_callback)
        reg1 = pyaiengine.Regex("Some URI", "^/$", uri_callback)
        reg2 = pyaiengine.Regex("HTTP Payload regex", "<HTML><HEAD>", payload_callback)

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        rman1 = pyaiengine.RegexManager([reg1])
        rman2 = pyaiengine.RegexManager([reg2])

        # Link to the DomainName d to the RegexManager for analise the uris
        dom.http_uri_regex_manager = rman1

        """ Link to the Regex another RegexManager """
        reg1.next_regex_manager = rman2

        self.stack.set_domain_name_manager(dman, "HTTP")

        self.inject("../pcapfiles/accessgoogle.pcap")

        self.assertEqual(self.called_callback, 3)
        self.assertEqual(dom.matchs, 1)

    def test67(self):
        """Verify that using the label for inject python code."""

        def domain_callback(flow):
            """ The code is executed on other time """
            exec(flow.label) in locals()
            self.called_callback += 1
            self.assertEqual(flow.label, "Hi change me!")

        def ipset_callback(flow):
            """ We use the label to put python code that later will be executed """
            flow.label = "flow.label=\"Hi change me!\""
            self.called_callback += 1

        iman = pyaiengine.IPSetManager()
        ipset = pyaiengine.IPSet("Generic set", ["74.125.24.99"], ipset_callback)
        dom = pyaiengine.DomainName("All HTTP", ".google.com", domain_callback)

        iman.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = iman

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTP")

        self.inject("../pcapfiles/accessgoogle.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(dom.matchs, 1)

    def test68(self):
        """Verify on SMTP access to mail data with regex."""

        def domain_callback(_):
            self.called_callback += 1

        def regex1_callback(flow):
            self.called_callback += 1
            cad = ""
            for i in flow.payload:
                cad += str(chr(i))
            self.assertGreater(cad.find("GCC"), 1000)

        def regex2_callback(_):
            self.called_callback += 1

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("Some r1", "^.*GCC.*$", regex1_callback)
        reg2 = pyaiengine.Regex("Some r2", "^.*(NextPart_000_0004_01CA45B0.095693F0).*$", \
            regex2_callback)
        dom = pyaiengine.DomainName("Some SMTP traffic", ".patriots.in", domain_callback)

        rman.add_regex(reg1)

        reg1.next_regex = reg2
        dom.regex_manager = rman
        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "SMTP")

        self.inject("../pcapfiles/smtp.pcap")

        self.assertEqual(self.called_callback, 3)
        self.assertEqual(dom.matchs, 1)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)

    def test69(self):
        """Verify that the data of SSH is correct."""

        self.stack.enable_protocol("ssh")

        adaptor = DatabaseTestAdaptor()
        self.stack.set_tcp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/ssh_flow.pcap")

        data = json.loads(adaptor.lastdata)
        # print(json.dumps(d,sort_keys=True,indent=4, separators=(",", ": ")))
        if "ssh" in data:
            self.assertEqual(data["ssh"]["clientname"], "SSH-2.0-Granados-2.0")
            self.assertEqual(data["ssh"]["servername"], "SSH-2.0-OpenSSH_5.3p1 Debian-3ubuntu3")
        else:
            self.assertTrue(False)

        self.assertEqual(len(self.stack.tcp_flow_manager), 1)
        self.assertEqual(len(self.stack.udp_flow_manager), 0)

    def test70(self):
        """Verify the quic traffic funcionality."""

        self.stack.enable_protocol("quic")

        adaptor = DatabaseTestAdaptor()
        self.stack.set_udp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/quic.pcap")

        data = json.loads(adaptor.lastdata)
        # print(json.dumps(d,sort_keys=True,indent=4, separators=(",", ": ")))

        if "quic" in data:
            self.assertEqual(data["quic"]["host"], "ad-emea.doubleclick.net")
            self.assertEqual(data["quic"]["ua"], "Chrome/52.0.2743.116 Linux x86_64")
        else:
            self.assertTrue(False)

        self.assertEqual(len(self.stack.tcp_flow_manager), 0)
        self.assertEqual(len(self.stack.udp_flow_manager), 1)

    def test71(self):
        """Verify the quic traffic funcionality with domain names."""

        def quic_callback(flow):
            self.called_callback += 1
            self.assertEqual(flow.quic_info.matched_domain_name, dom)
            self.assertEqual(flow.quic_info.host_name, "ad-emea.doubleclick.net")
            self.assertEqual(flow.quic_info.user_agent, "Chrome/52.0.2743.116 Linux x86_64")

        dom = pyaiengine.DomainName("Test", ".doubleclick.net")
        dom.callback = quic_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "Quic")

        self.inject("../pcapfiles/quic.pcap")

        self.assertEqual(self.called_callback, 1)

    def test72(self):
        """Verify the session ids on ssl traffic."""

        adaptor = DatabaseTestAdaptor()
        self.stack.set_tcp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/alibaba.pcap")
        session_id = "d7589284b1eecb5896a463486bace2a6569170c2e6bfa5e8329f93ef7806704c"

        data = json.loads(adaptor.lastdata)

        if "ssl" in data:
            self.assertEqual(data["ssl"]["session"], session_id)
        else:
            self.assertTrue(False)

        tcp_flows = self.stack.tcp_flow_manager
        self.assertEqual(len(tcp_flows), 2)

        host = "gw.alicdn.com"
        for flow in tcp_flows:
            self.assertEqual(session_id, flow.ssl_info.session_id)
            self.assertEqual(host, flow.ssl_info.server_name)

        total = [flow for flow in tcp_flows if flow.http_info]
        self.assertEqual(len(total), 0)

        total = [flow for flow in tcp_flows if flow.dns_info]
        self.assertEqual(len(total), 0)

        total = [flow for flow in tcp_flows if flow.ssl_info]
        self.assertEqual(len(total), 2)

    def test73(self):
        """Verify the retrieval of the interal caches with preallocated flows"""

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        cache = self.stack.get_cache("http", "host")

        self.assertIsNotNone(cache)
        self.assertEqual(cache.total_items, 1534)
        self.assertEqual(cache.total_acquires, 2)
        self.assertEqual(cache.total_releases, 0)
        self.assertEqual(cache.total_fails, 0)

        cache.destroy(1000)

        self.assertEqual(cache.total_items, 534)
        self.assertEqual(cache.total_acquires, 2)
        self.assertEqual(cache.total_releases, 0)
        self.assertEqual(cache.total_fails, 0)

        cache.reset()

        self.assertEqual(cache.total_acquires, 0)
        self.assertEqual(cache.total_releases, 0)
        self.assertEqual(cache.total_fails, 0)

        # Verify the output of the cache
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            cache.show()

        cache = self.stack.get_cache("http", "dontexist")

        self.assertIsNone(cache)

    def test74(self):
        """Verify the retrieval of the interal caches with preallocated flows"""

        cache_host = self.stack.get_cache("ssl", "host")
        self.assertIsNotNone(cache_host)
        cache_host.destroy(cache_host.total_items)

        cache_session = self.stack.get_cache("ssl", "session")
        self.assertIsNotNone(cache_session)
        cache_session.destroy(cache_session.total_items)

        self.inject("../pcapfiles/alibaba.pcap")
        for flow in self.stack.tcp_flow_manager:
            self.assertEqual("", flow.ssl_info.session_id)
            self.assertEqual("", flow.ssl_info.server_name)
            self.assertEqual("", flow.ssl_info.issuer_name)

        # Verify the output of the cache
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            cache_host.show()
            cache_session.show()

        cache = self.stack.get_cache("ssl", "dontexist")

        self.assertIsNone(cache)

    def test75(self):
        """Verify the retrieval of the internal coap caches,"""

        cache_host = self.stack.get_cache("coap", "host")
        self.assertIsNotNone(cache_host)
        cache_host.destroy(cache_host.total_items)

        cache_uri = self.stack.get_cache("coap", "uri")
        self.assertIsNotNone(cache_uri)
        cache_uri.destroy(cache_uri.total_items)

        self.inject("../pcapfiles/ipv4_coap.pcap")

        cache_host.dynamic_allocated_memory = True
        cache_uri.dynamic_allocated_memory = True
        self.stack.udp_flow_manager.flush()

        self.assertEqual(self.stack.tcp_flow_manager.flows, 0)
        self.assertEqual(self.stack.udp_flow_manager.flows, 0)

        self.inject("../pcapfiles/ipv4_coap.pcap")

        self.assertEqual(self.stack.udp_flow_manager.flows, 1)
        for flow in self.stack.udp_flow_manager:
            self.assertEqual("", flow.coap_info.host_name)
            self.assertEqual("/1/1/768/core.power", flow.coap_info.uri)

    def test76(self):
        """Verify the retrieval of the internal SSH caches."""

        cache_name = self.stack.get_cache("ssh", "name")
        self.assertIsNotNone(cache_name)
        cache_name.destroy(cache_name.total_items)

        self.stack.enable_protocol("ssh")

        adaptor = DatabaseTestAdaptor()
        self.stack.set_tcp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/ssh_flow.pcap")

        data = json.loads(adaptor.lastdata)
        if "ssh" in data:
            self.assertNotIn("clientname", data["ssh"])
            self.assertNotIn("servername", data["ssh"])
        else:
            self.assertTrue(False)

        self.assertEqual(len(self.stack.tcp_flow_manager), 1)
        self.assertEqual(len(self.stack.udp_flow_manager), 0)

    def test77(self):
        """Verify the retrieval of the internal SSDP caches."""

        cache_host = self.stack.get_cache("ssdp", "host")
        self.assertIsNotNone(cache_host)
        cache_host.destroy(cache_host.total_items)
        
        cache_uri = self.stack.get_cache("ssdp", "uri")
        self.assertIsNotNone(cache_uri)
        cache_uri.destroy(cache_uri.total_items)

        self.inject("../pcapfiles/ssdp_flow.pcap")

        # show the caches
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            cache_host.show()
            cache_uri.show()

        for flow in self.stack.udp_flow_manager:
            self.assertNotEqual(flow.ssdp_info, None)
            self.assertEqual(flow.ssdp_info.host_name, "")

    def test78(self):
        """Verify the retrieval of the internal IMAP cache."""

        def imap_callback(flow):
            # Nothing to retrieve there is no memory on the
            # cache allocated
            self.assertEqual(flow.imap_info.user_name, "")
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        dom = pyaiengine.DomainName("No trusted domain", "*", imap_callback)

        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "imap")
        self.stack.set_dynamic_allocated_memory(True)

        cache_user = self.stack.get_cache("imap", "user")
        self.assertIsNotNone(cache_user)
        cache_user.destroy(cache_user.total_items)
        cache_user.dynamic_allocated_memory = False

        self.inject("../pcapfiles/imap_flow.pcap")

        counters = self.stack.get_counters("imap")

        self.assertEqual(counters["commands"], 6)
        self.assertEqual(counters["responses"], 12)
        self.assertEqual(self.called_callback, 1)

    def test79(self):
        """Verify the retrieval of the netbios internal cache."""

        self.stack.link_layer_tag = "vlan"

        cache = self.stack.get_cache("netbios", "name")
        self.assertIsNotNone(cache)
        cache.destroy(cache.total_items)

        self.inject("../pcapfiles/flow_vlan_netbios.pcap")

        cache = self.stack.get_cache_data("netbios", "name")
        self.assertEqual(cache, {})

    def test80(self):
        """Verify the get of the pop internal cache object."""

        def pop_callback(flow):
            _ = flow.pop_info
            self.assertIsNotNone(flow.pop_info)
            # Nothing to retrieve there is no memory on the
            # cache allocated
            self.assertEqual(flow.pop_info.user_name, "")
            self.called_callback += 1

        dman = pyaiengine.DomainNameManager()
        dom = pyaiengine.DomainName("No trusted domain", "*", pop_callback)

        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "pop")

        cache = self.stack.get_cache("pop", "user")
        self.assertIsNotNone(cache)
        cache.destroy(cache.total_items)

        self.inject("../pcapfiles/pop_flow.pcap")

        counters = self.stack.get_counters("pop")

        self.assertEqual(counters["commands"], 15)
        self.assertEqual(counters["responses"], 16)
        self.assertEqual(self.called_callback, 1)

    def test81(self):
        """Verify the retrieval of the smb internal cache."""

        cache = self.stack.get_cache("smb", "filename")
        self.assertIsNotNone(cache)
        cache.destroy(cache.total_items)

        self.inject("../pcapfiles/smb_flow.pcap")

        counters = self.stack.get_counters("smb")

        self.assertEqual(counters["create files"], 8)
        
        for flow in self.stack.tcp_flow_manager:
            self.assertEqual(flow.smb_info.filename, "")

    def test82(self):
        """Retrive the internal smtp cache."""

        def domain_callback(flow):
            self.assertEqual(flow.smtp_info.mail_from, "")
            self.assertEqual(flow.smtp_info.mail_to, "")
            self.called_callback += 1

        dom = pyaiengine.DomainName("Some SMTP traffic", ".patriots.in", domain_callback)

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "SMTP")

        cache_from = self.stack.get_cache("smtp", "from")
        self.assertIsNotNone(cache_from)
        cache_from.destroy(cache_from.total_items)
        cache_from.dynamic_allocated_memory = False

        self.inject("../pcapfiles/smtp.pcap")

        self.assertEqual(dom.matchs, 1)

    def test83(self):
        """Verify the functionality of the multiple IPSets or IPRadixTrees
           on one IPSetManager and check the preference order.
           95.100.96.10, 95.100.96.48"""

        def ipset_first_callback(flow):
            ipset = flow.ip_set
            self.assertNotEqual(ipset, None)
            self.assertEqual(flow.dst_ip, "95.100.96.10")
            self.assertEqual(ipset.name, "First radix")
            self.called_callback += 1

        def ipset_second_callback(flow):
            ipset = flow.ip_set
            self.assertNotEqual(ipset, None)
            self.assertEqual(flow.dst_ip, "95.100.96.48")
            self.assertEqual(ipset.name, "Second radix")
            self.called_callback += 1

        # We will create a random list of IP address /16 to test
        # also the IPRadixTree
        ips = ["95.100.96.0/28", "192.172.12.1"]
        for _ in range(1000):
            ips.append("%d.%d.0.0/16" % (random.randint(96, 254), random.randint(1, 254)))

        first_rtree = pyaiengine.IPRadixTree("First radix", ips)
        second_rtree = pyaiengine.IPRadixTree("Second radix", ["95.100.96.0/26", "192.172.12.100"])
        first_ipset = pyaiengine.IPSet("First set", ["1.1.1.1", "2.2.2.2"])
        second_ipset = pyaiengine.IPSet("Second set", ["95.100.96.10"])

        im = pyaiengine.IPSetManager()

        # The order of evaluation will be the following one
        im.add_ip_set(first_rtree)
        im.add_ip_set(first_ipset)
        im.add_ip_set(second_ipset)
        im.add_ip_set(second_rtree)

        first_rtree.callback = ipset_first_callback
        second_rtree.callback = ipset_second_callback

        self.stack.tcp_ip_set_manager = im

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(first_rtree.lookups, 2)
        self.assertEqual(first_rtree.lookups_in, 1)
        self.assertEqual(first_rtree.lookups_out, 1)

        self.assertEqual(first_ipset.lookups, 1)
        self.assertEqual(first_ipset.lookups_in, 0)
        self.assertEqual(first_ipset.lookups_out, 1)

        self.assertEqual(second_ipset.lookups, 1)
        self.assertEqual(second_ipset.lookups_in, 0)
        self.assertEqual(second_ipset.lookups_out, 1)

        self.assertEqual(second_rtree.lookups, 1)
        self.assertEqual(second_rtree.lookups_in, 1)
        self.assertEqual(second_rtree.lookups_out, 0)

        self.assertEqual(self.called_callback, 2)

    def test84(self):
        """Verify the method flush from the FlowManager object."""

        def exists(table, property):
            for flow in table:
                if getattr(flow, property):
                    return True
            return False

        self.inject("../pcapfiles/alibaba.pcap")
        self.inject("../pcapfiles/smtp.pcap")
        self.inject("../pcapfiles/pop_flow.pcap")

        # Now in memory there is 2 ssl flows, 1 pop and one smtp
        self.assertFalse(exists(self.stack.tcp_flow_manager, "dns_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "ssl_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "pop_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "smtp_info"))

        self.stack.tcp_flow_manager.flush("smtp")

        self.assertTrue(exists(self.stack.tcp_flow_manager, "ssl_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "pop_info"))
        self.assertFalse(exists(self.stack.tcp_flow_manager, "smtp_info"))

        self.stack.tcp_flow_manager.flush("tcpgeneric")

        self.assertTrue(exists(self.stack.tcp_flow_manager, "ssl_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "pop_info"))
        self.assertFalse(exists(self.stack.tcp_flow_manager, "smtp_info"))

        self.stack.tcp_flow_manager.flush("ssl")

        self.assertFalse(exists(self.stack.tcp_flow_manager, "ssl_info"))
        self.assertTrue(exists(self.stack.tcp_flow_manager, "pop_info"))
        self.assertFalse(exists(self.stack.tcp_flow_manager, "smtp_info"))

        self.stack.tcp_flow_manager.flush()

        self.assertEqual(len(self.stack.tcp_flow_manager), 0)

    def test85(self):
        """Verify that exceptions on callbacks disable the callback."""

        def bad_callback(flow):
            self.called_callback += 1
            _ = generate_a_exception_variable

        dom = pyaiengine.DomainName("Alibaba test", ".alicdn.com")
        dom.callback = bad_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "ssl")

        devnull = open(os.devnull, "w")
        original_stderr = sys.stderr
        sys.stderr = devnull 
        self.inject("../pcapfiles/alibaba.pcap")
        devnull.close()
        sys.stderr = original_stderr

        self.assertEqual(self.called_callback, 1)

    def test86(self):
        """Check the function reset_counters works as expected."""

        self.inject("../pcapfiles/accessgoogle.pcap")
        
        counters = self.stack.get_counters("DNS")
        self.assertEqual(counters["queries"], 2)
        self.assertEqual(counters["responses"], 2)

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["requests"], 2)
        self.assertEqual(counters["responses"], 2)
        self.assertEqual(counters["gets"], 2)

        self.stack.reset_counters("dns")

        counters = self.stack.get_counters("DNS")
        self.assertEqual(counters["queries"], 0)
        self.assertEqual(counters["responses"], 0)

        self.stack.reset_counters("http")

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["requests"], 0)
        self.assertEqual(counters["responses"], 0)
        self.assertEqual(counters["gets"], 0)

        self.inject("../pcapfiles/alibaba.pcap")

        counters = self.stack.get_counters("ssL")
        self.assertEqual(counters["handshakes"], 4)
        self.assertEqual(counters["records"], 14)

        self.stack.reset_counters("SSL")

        counters = self.stack.get_counters("ssL")
        self.assertEqual(counters["handshakes"], 0)
        self.assertEqual(counters["records"], 0)

        self.inject("../pcapfiles/smtp.pcap")

        counters = self.stack.get_counters("SMTP")
        self.assertEqual(counters["hellos"], 7)
        self.assertEqual(counters["responses"], 10)

        self.stack.reset_counters("smtp")

        counters = self.stack.get_counters("SMTP")
        self.assertEqual(counters["hellos"], 0)
        self.assertEqual(counters["responses"], 0)

        self.inject("../pcapfiles/pop_flow.pcap")

        counters = self.stack.get_counters("pOp")
        self.assertEqual(counters["commands"], 15)
        self.assertEqual(counters["responses"], 16)
        self.assertEqual(counters["stats"], 4)
        self.assertEqual(counters["lists"], 4)
        self.assertEqual(counters["retrs"], 20)
        self.assertEqual(counters["deletes"], 20)

        self.stack.reset_counters("pop")

        counters = self.stack.get_counters("pOp")
        self.assertEqual(counters["commands"], 0)
        self.assertEqual(counters["responses"], 0)
        self.assertEqual(counters["stats"], 0)
        self.assertEqual(counters["lists"], 0)
        self.assertEqual(counters["retrs"], 0)
        self.assertEqual(counters["deletes"], 0)

    def test87(self):
        """Test the stack with DTLS traffic."""

        self.inject("../pcapfiles/ipv4_dtls.pcap")

        counters = self.stack.get_counters("dtls")

        self.assertEqual(counters["server hellos"], 1)
        self.assertEqual(counters["server dones"], 1)
        self.assertEqual(counters["records"], 11)
        self.assertEqual(counters["packets"], 5)
        self.assertEqual(counters["alerts"], 0)
        self.assertEqual(counters["encrypt handshakes"], 3)

        self.stack.reset_counters("dtls")
        counters = self.stack.get_counters("dtls")

        self.assertEqual(counters["server hellos"], 0)
        self.assertEqual(counters["server dones"], 0)
        self.assertEqual(counters["records"], 0)
        self.assertEqual(counters["packets"], 0)
        self.assertEqual(counters["alerts"], 0)
        self.assertEqual(counters["encrypt handshakes"], 0)

    def test88(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [192.168.1.13:45726]:6:[74.125.24.99:80]
        [192.168.1.13:54737]:17:[89.101.160.5:53]"""
 
        self.inject("../pcapfiles/accessgoogle.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.13:45726]:6:[74.125.24.99:80]", data)
            self.assertIn(b"[192.168.1.13:54737]:17:[89.101.160.5:53]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_port=80)
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.13:45726]:6:[74.125.24.99:80]", data)
            self.assertNotIn(b"[192.168.1.13:54737]:17:[89.101.160.5:53]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_port=80, protocol=6, l7protocol="HTTP")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.13:45726]:6:[74.125.24.99:80]", data)
            self.assertNotIn(b"[192.168.1.13:54737]:17:[89.101.160.5:53]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_port=53, protocol=17, l7protocol="DNS")
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[192.168.1.13:45726]:6:[74.125.24.99:80]", data)
            self.assertIn(b"[192.168.1.13:54737]:17:[89.101.160.5:53]", data)

    def test89(self):
        """Test the learner and frequencies with all flows
        and use the generated regex expression."""

        learner = pyaiengine.LearnerEngine()
        freq_tcp = pyaiengine.FrequencyGroup()

        self.stack.mode = "frequency"

        self.inject("../pcapfiles/alibaba.pcap")

        # Add all the TCP Flows of the FlowManager on the FrequencyEngine
        freq_tcp.add_all_flows(self.stack.tcp_flow_manager)
        freq_tcp.compute()

        self.assertEqual(freq_tcp.total_computed_frequencies, 1)
        self.assertEqual(freq_tcp.total_process_flows, 2)

        flow_list = freq_tcp.get_reference_flows()
        learner.agregate_flows(flow_list)
        learner.compute()

        # Now we use the generated regular expression
        self.stack.mode = "nids"
        self.stack.tcp_flow_manager.flush()

        rman = pyaiengine.RegexManager()
        short_regex = "^\\x16\\x03\\x01\\x02\\x00\\x01\\x00\\x01\\xfc\\x03\\x03"
        reg = pyaiengine.Regex("automatic regex", learner.regex)

        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.assertIn(short_regex, learner.regex)
        self.inject("../pcapfiles/alibaba.pcap")

        self.assertEqual(reg.matchs, 2)

    def test90(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [10.42.0.211:51057]:6:[64.71.142.82:443]
        [10.42.0.211:38380]:6:[64.71.142.110:443]"""

        self.inject("../pcapfiles/alibaba.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(protocol=6)
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.42.0.211:51057]:6:[64.71.142.82:443]", data)
            self.assertIn(b"[10.42.0.211:38380]:6:[64.71.142.110:443]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="64.71.142.0/24")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.42.0.211:51057]:6:[64.71.142.82:443]", data)
            self.assertIn(b"[10.42.0.211:38380]:6:[64.71.142.110:443]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="64.71.142.0/25")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.42.0.211:51057]:6:[64.71.142.82:443]", data)
            self.assertIn(b"[10.42.0.211:38380]:6:[64.71.142.110:443]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="64.71.142.82/26")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.42.0.211:51057]:6:[64.71.142.82:443]", data)
            self.assertIn(b"[10.42.0.211:38380]:6:[64.71.142.110:443]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="64.71.142.82/27")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.42.0.211:51057]:6:[64.71.142.82:443]", data)
            self.assertNotIn(b"[10.42.0.211:38380]:6:[64.71.142.110:443]", data)

    def test91(self):
        """ Verify the functionality of the HTTPUriSets with * """

        self.called_callback1 = 0
        self.called_callback2 = 0

        def uri_callback1(flow):
            self.called_callback += 1
            self.called_callback1 += 1

        def uri_callback2(flow):
            self.called_callback += 1
            self.called_callback2 += 1

        uset1 = pyaiengine.HTTPUriSet()
        uset2 = pyaiengine.HTTPUriSet()
        uset1.callback = uri_callback1
        uset2.callback = uri_callback2

        dom1 = pyaiengine.DomainName("Wired domain", ".wired.com")
        dom2 = pyaiengine.DomainName("Wired domain", "ds.serving-sys.com")

        uset2.add_uri("/BurstingRes///Site-2263/Type-0/7041dd49-4b44-40ab-9645-42538292f9f6.jpg")
        uset1.add_uri("*")

        dom1.http_uri_set = uset1
        dom2.http_uri_set = uset2

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom1)
        dman.add_domain_name(dom2)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.called_callback2, 1)
        self.assertEqual(self.called_callback1, 39)

        self.assertEqual(uset1.lookups, 39)
        self.assertEqual(uset1.lookups_in, 39)
        self.assertEqual(uset1.lookups_out, 0)

        self.assertEqual(uset2.lookups, 3)
        self.assertEqual(uset2.lookups_in, 1)
        self.assertEqual(uset2.lookups_out, 2)

        self.assertEqual(self.called_callback, 39 + 1)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            uset2.show() 
            file.seek(0)
            data = file.read()
            self.assertIn(b"/BurstingRes///Site-2263/Type-0/7041dd49-4b44-40ab-9645-42538292f9f6.jpg", data)

    def test92(self):
        """Verify the functionality when there is a cache fail."""

        self.called_callback = 0

        def fail_cache_callback(flow):
            self.called_callback += 1
            self.assertEqual("192.168.1.13:44265:6:74.125.24.189:443", str(flow))

        # Reduce the memory for store SSL information
        self.stack.decrease_allocated_memory("ssl", 1000)

        # Set the callback for be called on a cache fail
        self.stack.set_on_fail_cache_callback("ssl", fail_cache_callback)

        self.inject("../pcapfiles/sslflow.pcap")

        self.assertEqual(self.called_callback , 1)

    def test93(self):
        """Verify that SSL counters with MTLS are correct."""

        self.inject("../pcapfiles/mtls_flow.pcap")

        data = {"packets": 6, "bytes": 4511, "allow hosts": 1, "banned hosts": 0,
                "handshakes": 6, "encrypt handshakes": 1, "alerts": 1, "change cipher specs": 1,
                "datas": 0, "client hellos": 1, "server hellos": 1, "certificates": 2,
                "server key exchanges": 0, "certificate requests": 0, "server dones": 0,
                "certificate verifies": 1, "client key exchanges": 1, "handshake dones": 0, 
                "new session tickets": 0, "records": 9}

        counters = self.stack.get_counters("SSL")
        self.assertDictEqual(counters, data)

    def test94(self):
        """Verify that a HTTP requests with no Host field matchs
        with a DomainName * """

        self.called_callback = 0

        def callback(flow):
            self.called_callback += 1

        dom = pyaiengine.DomainName("All domains", "*", callback)

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("./protocols/http/packets/packet04.pcap")

        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)

        dman.remove_domain_name("All domains")
        dom = pyaiengine.DomainName("All domains", ".com", callback)
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("./protocols/http/packets/packet04.pcap")

        self.assertEqual(self.called_callback, 1)
        self.assertEqual(dom.matchs, 0)

    def test95(self):
        """Verify that the "data time" is working."""

        self.inject("./protocols/http/packets/packet04.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show("http", 2)
            file.seek(0)
            data = file.read()
            self.assertIn(b"Data time:             00:00:00:00", data)

        time.sleep(1)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show("http", 2)
            file.seek(0)
            data = file.read()
            self.assertIn(b"Data time:             00:00:00:01", data)

        self.stack.release_cache("http")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show("http", 2)
            file.seek(0)
            data = file.read()
            self.assertIn(b"Data time:             00:00:00:00", data)



class StackMobileTests(unittest.TestCase):
    """This class handles the test cases for mobile enviroments
    such as GPRS, GTP tunneling."""

    def setUp(self):
        self.stack = pyaiengine.StackMobile()
        self.pdis = pyaiengine.PacketDispatcher()
        self.pdis.stack = self.stack
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile):
        """Method for process the pcapfile."""

        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """Verify the integrity of the sip fields."""

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.mqtt_info, None)
            self.assertEqual(flow.coap_info, None)
            self.assertEqual(flow.http_info, None)
            self.assertEqual(flow.dns_info, None)
            self.assertEqual(flow.ssl_info, None)
            self.assertNotEqual(flow.sip_info, None)
            self.assertEqual(flow.sip_info.from_name, \
                "\"User1\" <sip:ng40user1@apn.sip.voice.ng4t.com>;tag=690711")
            self.assertEqual(flow.sip_info.to_name, \
               "\"User1\" <sip:ng40user1@apn.sip.voice.ng4t.com>")
            self.assertEqual(flow.sip_info.uri, "sip:10.255.1.111:5090")
            self.assertEqual(flow.sip_info.via, \
                "SIP/2.0/UDP 10.255.1.1:5090;branch=z9hG4bK199817980098801998")

        data = json.loads(adaptor.lastdata)
        counters = self.stack.get_counters("SIP")
        self.assertEqual(counters["requests"], 7)
        self.assertEqual(counters["responses"], 7)
        self.assertEqual(counters["registers"], 2)

        if "sip" in data:
            self.assertEqual(data["sip"]["uri"], "sip:apn.sip.voice.ng4t.com")
            self.assertEqual(data["sip"]["from"], \
                "\"User1\" <sip:ng40user1@apn.sip.voice.ng4t.com>;tag=690711")
            self.assertEqual(data["sip"]["to"], \
                "\"User1\" <sip:ng40user1@apn.sip.voice.ng4t.com>")
            self.assertEqual(data["sip"]["via"], \
                "SIP/2.0/UDP 10.255.1.1:5090;branch=z9hG4bK199817980098801998")
        else:
            self.assertTrue(False)

        data_cache = self.stack.get_cache_data("SIP", "uri")

        cache1 = {"sip:10.255.1.111:5090": 2, "sip:apn.sip.voice.ng4t.com": 4,
                  "sip:ng40user11@apn.sip.voice.ng4t.com": 1}

        self.assertDictEqual(data_cache, cache1)

        self.stack.release_cache("SIP")

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.sip_info, None)

    def test02(self):
        """Test some regex on the mobile stack."""

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("r1", b"^\\x58\\x67\\x77\\x86.*$")
        reg2 = pyaiengine.Regex("r2", b"^.*\\xde\\xed\\xfc\\x0c.*$")

        rman.add_regex(reg1)

        reg1.next_regex = reg2

        self.assertEqual(sys.getrefcount(reg2), 3)

        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/gprs_ftp.pcap")

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)

        ft = self.stack.tcp_flow_manager
        self.assertEqual(len(ft), 1)
        ft.flush()
        self.assertEqual(len(ft), 0)

        # Unset the next regex of reg1
        reg1.next_regex = None
        self.assertEqual(sys.getrefcount(reg2), 2)

        self.inject("../pcapfiles/gprs_ftp.pcap")

        self.assertEqual(reg1.matchs, 2)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg1.next_regex, None)
        self.assertEqual(reg2.next_regex, None)

    def test03(self):
        """ Tests the database adaptor on tcp and callbacks on regex """

        def callback1(flow):
            self.called_callback += 1
            self.assertNotEqual(flow.regex, None)
            self.assertEqual(flow.regex.name, "r1")

        def callback2(flow):
            self.assertNotEqual(flow.regex, None)
            self.assertEqual(flow.regex.name, "r2")
            self.called_callback += 1

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor)

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("r1", b"^\\x58\\x67\\x77\\x86.*$", callback1)
        reg2 = pyaiengine.Regex("r2", b"^.*\\xde\\xed\\xfc\\x0c.*$", callback2)

        rman.add_regex(reg1)
        reg1.next_regex = reg2

        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/gprs_ftp.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)

        data = json.loads(adaptor.lastdata)
        self.assertEqual(data["matchs"], "r2")

    def test04(self):
        """ disable SIP protocol and check values """

        self.stack.set_dynamic_allocated_memory(True)
        self.stack.enable_protocol("sip")

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        counters = self.stack.get_counters("SIP")
        self.assertEqual(counters["requests"], 7)
        self.assertEqual(counters["responses"], 7)
        self.assertEqual(counters["registers"], 2)
        self.assertEqual(counters["packets"], 22)
        self.assertEqual(counters["bytes"], 14537)

        counters = self.stack.get_counters("udpgeneric")
        self.assertEqual(counters["packets"], 0)
        self.assertEqual(counters["bytes"], 0)

        self.stack.udp_flow_manager.flush()
        self.stack.tcp_flow_manager.flush()

        self.stack.disable_protocol("sip")

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        counters = self.stack.get_counters("SIP")
        self.assertEqual(counters["requests"], 7)
        self.assertEqual(counters["responses"], 7)
        self.assertEqual(counters["registers"], 2)

        counters = self.stack.get_counters("udpgeneric")
        self.assertEqual(counters["packets"], 22)
        self.assertEqual(counters["bytes"], 14537)
       
        counters = self.stack.get_counters("gprs")
        self.assertEqual(counters["tpdus"], 44)

        self.stack.reset_counters("sip")
        self.stack.reset_counters("gprs")

        counters = self.stack.get_counters("SIP")
        self.assertEqual(counters["requests"], 0)
        self.assertEqual(counters["responses"], 0)
        self.assertEqual(counters["registers"], 0)

        counters = self.stack.get_counters("gprs")
        self.assertEqual(counters["tpdus"], 0)

    def test05(self):
        """Verify the integrity of the sip internal caches."""

        # Remove all the memory for store uris
        cache = self.stack.get_cache("SIP", "uri")
        cache.destroy(cache.total_items)

        # Remove all the memory for store froms
        cache = self.stack.get_cache("sIp", "from")
        cache.destroy(cache.total_items)

        # Remove all the memory for store tos
        cache = self.stack.get_cache("SIP", "to")
        cache.destroy(cache.total_items)

        # Remove all the memory for store vias
        cache = self.stack.get_cache("SIP", "via")
        cache.destroy(cache.total_items)

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        for flow in self.stack.udp_flow_manager:
            self.assertNotEqual(flow.sip_info, None)
            self.assertEqual(flow.sip_info.from_name, "")
            self.assertEqual(flow.sip_info.to_name, "")
            self.assertEqual(flow.sip_info.uri, "")
            self.assertEqual(flow.sip_info.via, "")

    def test06(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [10.1.2.11:2152]:17:[10.1.1.12:2152]
        [10.255.1.1:5090]:17:[10.0.0.100:5060]"""

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="gprs")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertNotIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="sip", dst_port=5555)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertNotIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="sip", src_port=5090)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="10.0.0.0/8")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="10.0.0.0/16")
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[10.1.2.11:2152]:17:[10.1.1.12:2152]", data)
            self.assertIn(b"[10.255.1.1:5090]:17:[10.0.0.100:5060", data)

    @unittest.skipIf(not defined("HAVE_PCAP_TIMERS"), "Test not supported")
    def test07(self):
        """Verify the timers processing pcap files."""

        def timer_call():
            self.called_callback += 1  

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        with pyaiengine.PacketDispatcher("../pcapfiles/gprs_sip_flow.pcap") as pdis:
            pdis.stack = self.stack
            pdis.add_timer(timer_call, 1)
            pdis.run()

        self.assertEqual(self.called_callback, 9)

        self.called_callback = 0

        with pyaiengine.PacketDispatcher("../pcapfiles/gprs_sip_flow.pcap") as pdis:
            pdis.stack = self.stack
            pdis.add_timer(timer_call, 4)
            pdis.run()

        self.assertEqual(self.called_callback, 2)

    def test08(self):
        """Verify the functionality when there is a cache fail."""

        self.called_callback = 0

        def fail_cache_callback(flow):
            self.called_callback += 1
            self.assertEqual("10.255.1.1:5090:17:10.0.0.100:5060", str(flow))

        # Reduce the memory for store SIP information
        self.stack.decrease_allocated_memory("sip", 1000)

        # Set the callback for be called on a cache fail
        self.stack.set_on_fail_cache_callback("sip", fail_cache_callback)

        self.inject("../pcapfiles/gprs_sip_flow.pcap")

        self.assertEqual(self.called_callback , 1)


class StackLanIPv6Tests(unittest.TestCase):
    """Class for handle the test cases related to IPv6."""

    def setUp(self):
        self.stack = pyaiengine.StackLanIPv6()
        self.pdis = pyaiengine.PacketDispatcher()
        self.pdis.stack = self.stack
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile):
        """Method for process the pcapfile."""
        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """Create a regex for a generic exploit."""

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"
        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("generic exploit", b"^.*\\x0c\\x0a\\x15\\x61\\x47\\x59\\x96.*$")
        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(reg.matchs, 1)

    def test02(self):
        """Create a regex for a generic exploit and a IPSet."""

        def ipset_callback(flow):
            self.assertIsNotNone(flow)

            # Check some TCP Flags
            # Bear in mind that the ipsets are trigger
            # on the first packet
            tcp = flow.tcp_info
            self.assertIsNotNone(tcp)
            self.assertEqual(tcp.syns, 1)
            self.assertEqual(tcp.synacks, 0)
            self.assertEqual(tcp.synacks, 0)
            self.assertEqual(tcp.acks, 0)
            self.assertEqual(tcp.fins, 0)
            self.assertEqual(tcp.rsts, 0)
            self.assertEqual(tcp.pushs, 0)
            self.assertEqual(tcp.state, "SYN_SENT")

            self.called_callback += 1

        ipset = pyaiengine.IPSet("IPv6 generic set", ["2003:de:2016:110::b15:22"])
        ipset.add_ip_address("this is not an ip")
        ipset.add_ip_address("bbbbNOIPdc20:c7f:2012:11::1")
        ipset.add_ip_address("192.168.1.1")
        ipset.callback = ipset_callback
        im = pyaiengine.IPSetManager()

        # There is only two valid IP address 
        self.assertEqual(len(ipset), 2)

        im.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = im

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("generic exploit", b"^.*\\x0c\\x0a\\x15\\x61\\x47\\x59\\x96.*$")
        rman.add_regex(reg1)
        reg2 = pyaiengine.Regex("other exploit", "(this can not match)")
        rman.add_regex(reg2)
        self.stack.tcp_regex_manager = rman

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(self.called_callback, 1)

        ipset.remove_ip_address("dc20:c7f:2012:11::2")

    def test03(self):
        """Create a regex for a generic exploit and a IPSet with no matching."""

        def ipset_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        ips = ["dc20:c7f:2012:11::22", "dc20:c7f:2012:11::1"]

        ipset = pyaiengine.IPSet(ips)
        ipset.callback = ipset_callback
        iman = pyaiengine.IPSetManager()

        iman.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = iman

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("generic exploit", b"^.*\\x00\\x0a\\x15\\x61\\x47\\x59\\x96.*$")
        rman.add_regex(reg1)
        reg2 = pyaiengine.Regex("other exploit", "(this can not match)")
        rman.add_regex(reg2)
        self.stack.tcp_regex_manager = rman

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(reg1.matchs, 0)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(self.called_callback, 0)

    def test04(self):
        """Attach a database to the engine for TCP traffic."""

        dbaux = DatabaseTestAdaptor()
        adaptor = DatabaseTestAdaptor()

        self.assertEqual(sys.getrefcount(adaptor), 2)
        self.assertEqual(sys.getrefcount(dbaux), 2)

        self.stack.set_tcp_database_adaptor(adaptor, 16)
        self.stack.set_tcp_database_adaptor(dbaux, 16)
        self.assertEqual(sys.getrefcount(adaptor), 2)
        self.assertEqual(sys.getrefcount(dbaux), 3)

        self.stack.link_layer_tag = "vlan"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(dbaux.total_inserts, 1)
        self.assertEqual(dbaux.total_updates, 4)
        self.assertEqual(dbaux.total_removes, 0)

        self.assertEqual(sys.getrefcount(dbaux), 3)
        # Check Protocol.cc there is a bug with python3.5 needs investigation
        # self.stack.set_tcp_database_adaptor(None)
        # self.assertEqual(sys.getrefcount(adaptor), 2)

    def test05(self):
        """Attach a database to the engine for UDP traffic."""

        db_udp = DatabaseTestAdaptor()
        db_tcp = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(db_udp, 16)
        self.stack.set_tcp_database_adaptor(db_tcp)

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        self.assertEqual(db_udp.total_inserts, 1)
        self.assertEqual(db_udp.total_updates, 1)
        self.assertEqual(db_udp.total_removes, 0)

        self.assertEqual(db_tcp.total_inserts, 0)
        self.assertEqual(db_tcp.total_updates, 0)
        self.assertEqual(db_tcp.total_removes, 0)

    def test06(self):
        """Several IPSets with no matching."""

        def ipset_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        ipset1 = pyaiengine.IPSet("IPSet 1", ["dcbb:c7f:2012:11::22"])
        ipset2 = pyaiengine.IPSet("IPSet 2", ["dcaa:c7f:2012:11::22"])
        ipset3 = pyaiengine.IPSet("IPSet 3", ["dc20:c7f:2012:11::2"])

        iman = pyaiengine.IPSetManager()

        iman.add_ip_set(ipset1)
        iman.add_ip_set(ipset2)
        iman.add_ip_set(ipset3)

        self.stack.tcp_ip_set_manager = iman

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(len(iman), 3)
        self.assertEqual(self.called_callback, 0)
        self.assertEqual(self.stack.tcp_ip_set_manager, iman)

    def test07(self):
        """Extract IPv6 address from a DomainName matched and IPSet functionality."""

        def dns_callback(flow):
            for ip in flow.dns_info:
                if ip == "2607:f8b0:4001:c05::6a":
                    self.called_callback += 1
            self.assertEqual(flow.dns_info.matched_domain_name, dom)
            self.assertEqual(flow.dns_info.query_type, 28)
            self.assertEqual(flow.dns_info.rcode, 0)

            # This flow is UDP so no tcp info on the flow
            self.assertIsNone(flow.tcp_info)

        def ipset_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        dom = pyaiengine.DomainName("Google test", ".google.com")
        dom.callback = dns_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        ipset1 = pyaiengine.IPSet("IPSet 1", ["2001:abcd::1"], ipset_callback)

        iman = pyaiengine.IPSetManager()
        iman.add_ip_set(ipset1)

        self.stack.udp_ip_set_manager = iman
        self.assertEqual(self.stack.udp_ip_set_manager, iman)

        self.stack.set_domain_name_manager(dman, "dns")

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        self.assertEqual(ipset1.lookups, 1)
        self.assertEqual(ipset1.lookups_in, 1)
        self.assertEqual(ipset1.lookups_out, 0)
        self.assertEqual(self.called_callback, 2)

        # Verify some of the counters of the dns protocol

        counters = self.stack.get_counters("dns")
        self.assertEqual(counters["queries"], 1)
        self.assertEqual(counters["allow queries"], 1)
        self.assertEqual(counters["banned queries"], 0)
        self.assertEqual(counters["responses"], 1)
        self.assertEqual(counters["type AAAA"], 1)

        for ipset in iman:
            _ = ipset

    def test08(self):
        """Test the functionality of make graphs of regex, for complex detecctions."""

        rmbase = pyaiengine.RegexManager()
        rman2 = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("r1", b"^SSH.*$")

        rmbase.add_regex(reg1)

        reg1.next_regex_manager = rman2

        reg2 = pyaiengine.Regex("reg2", b"(this can not match)")
        reg3 = pyaiengine.Regex("reg3", b"^.*(curve25519).*$")
        rman2.add_regex(reg2)
        rman2.add_regex(reg3)

        self.stack.tcp_regex_manager = rmbase

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(reg3.matchs, 1)

    def test09(self):
        """ Another test for the functionality of make graphs of regex, for complex detecctions """

        rman1 = pyaiengine.RegexManager()
        rman2 = pyaiengine.RegexManager()
        rman3 = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("reg1", b"^(SSH).*$")

        reg1.next_regex_manager = rman2
        rman1.add_regex(reg1)

        reg2 = pyaiengine.Regex("reg2", b"(this can not match)")
        reg3 = pyaiengine.Regex("reg3", b"^(SSH).*$")
        rman2.add_regex(reg2)
        rman2.add_regex(reg3)

        reg3.next_regex_manager = rman3

        reg4 = pyaiengine.Regex("reg4", b"^.*(curve25519).*$")
        reg5 = pyaiengine.Regex("reg5", b"(this can not match)")

        rman3.add_regex(reg4)
        rman3.add_regex(reg5)

        self.stack.tcp_regex_manager = rman1

        oldstack = None

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")
        oldstack = self.stack

        self.assertEqual(self.stack, oldstack)

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 0)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(reg4.matchs, 1)

    def test10(self):
        """ Verify the functionality of the get_cache_data method """

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        data = self.stack.get_cache_data("DNS", "name")
        self.assertEqual(len(self.stack.get_cache_data("DNS", "name")), 1)
        self.assertEqual(len(self.stack.get_cache_data("DNSNoExists", "name")), 0)
        self.stack.release_cache("DNS")
        self.assertEqual(len(self.stack.get_cache_data("DNS", "name")), 0)
        self.assertEqual(len(self.stack.get_cache_data("HTTP", "host")), 0)
        self.assertEqual(len(self.stack.get_cache_data("SSL", "host")), 0)

        self.assertIsNotNone(data["www.google.com"])

    def test11(self):
        """ Verify the correctness of the HTTP Protocol on IPv6 """

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["requests"], 11)
        self.assertEqual(counters["responses"], 11)

        data = self.stack.get_cache_data("http", "host")
        self.assertIsNotNone(data["media.us.listen.com"])

    def test12(self):
        """ Verify the functionality of the RegexManager on the HTTP Protocol for analise
            inside the l7 payload of HTTP on IPv6 traffic """

        def callback_domain(flow):
            # These are the ttls when the domain rule is trigger
            self.assertEqual(flow.upstream_ttl, 255)
            self.assertEqual(flow.downstream_ttl, 236)
            self.called_callback += 1

        def callback_regex(flow):
            self.called_callback += 1
            self.assertEqual(flow.regex.name, "Regex for analysing the content of HTTP")
            self.assertEqual(flow.http_info.host_name, "media.us.listen.com")

        dom = pyaiengine.DomainName("Music domain", ".us.listen.com")

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("Regex for analysing the content of HTTP", \
            b"^\\x89\\x50\\x4e\\x47\\x0d\\x0a\\x1a\\x0a.*$")

        rman.add_regex(reg1)
        reg1.callback = callback_regex

        # So the flows from listen.com will be analise the regexmanager attached
        dom.regex_manager = rman

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "http")

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        self.assertEqual(self.called_callback, 2)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(dom.matchs, 1)

    def test13(self):
        """ Verify the functionality of the Evidence manager with IPv6 and UDP """

        def domain_callback(flow):
            self.called_callback += 1
            flow.evidence = True

        dom = pyaiengine.DomainName("Google domain", ".google.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = domain_callback
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "dns")

        with pyaiengine.PacketDispatcher("../pcapfiles/ipv6_google_dns.pcap") as pdis:
            pdis.evidences = True
            pdis.stack = self.stack
            pdis.run()

        self.assertEqual(self.called_callback, 1)
        self.assertEqual(dom.matchs, 1)

        # verify the integrity of the new file created
        files = glob.glob("evidences.*.pcap")
        os.remove(files[0])

    def test14(self):
        """
        Verify the functionality write on the databaseAdaptor
        when a important event happen on TCP.
        """

        reg = pyaiengine.Regex("my regex", b"^.*(curve25519).*$")

        # Force to write the packet
        reg.write_packet = True

        rman = pyaiengine.RegexManager([reg])

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor)

        self.stack.tcp_regex_manager = rman

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        data = json.loads(adaptor.lastdata)
        if "matchs" in data:
            self.assertEqual(data["matchs"], "my regex")
        else:
            self.assertTrue(False)

        # the packet is write on the packet field of the json
        self.assertEqual(reg.write_packet, True)

    def test15(self):
        """ Verify the flush functionality of the FlowManager """

        # increase the timeout of the flows because the difference between the pcaps
        # is more than one year
        self.stack.flows_timeout = 60 * 60 * 500

        self.inject("../pcapfiles/ipv6_google_dns.pcap")
        self.inject("../pcapfiles/http_over_ipv6.pcap")

        udp_flows = self.stack.udp_flow_manager
        tcp_flows = self.stack.tcp_flow_manager

        self.assertEqual(len(udp_flows), 1)
        self.assertEqual(len(tcp_flows), 1)

        udp_flows.flush()

        self.assertEqual(len(udp_flows), 0)
        self.assertEqual(len(tcp_flows), 1)

        tcp_flows.flush()

        self.assertEqual(len(udp_flows), 0)
        self.assertEqual(len(tcp_flows), 0)

    def test16(self):
        """ Test the callbacks on the RegexManager for TCP traffic """

        def callback_re(flow):
            self.fail("shouldn't happen")

        def callback_rm(flow):
            self.assertEqual(flow.regex.matchs, 1)
            self.assertEqual(flow.regex.name, "generic exploit")
            self.called_callback += 1

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("generic exploit", b"^.*(curve25519).*$")
        reg.callback = callback_re
        rman.callback = callback_rm

        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(self.called_callback, 1)
        self.assertEqual(reg.matchs, 1)

    def test17(self):
        """Extract IPv6 address from a DomainName matched on a adaptor."""

        dom = pyaiengine.DomainName("Google test", ".google.com")

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        adaptor = DatabaseTestAdaptor()
        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        # There is no domainname manager attached so the ips should not be populated
        data = json.loads(adaptor.lastdata)

        if "dns" in data:
            if "ips" in data["dns"]:
                self.fail("The IPs must not be on the data")
            else:
                self.assertTrue(True)
        else:
            self.fail("The DNS must be on the data")

        self.stack.udp_flow_manager.flush()

        self.stack.set_domain_name_manager(dman, "dns")
        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        # The ips should be on the ips secction
        data = json.loads(adaptor.lastdata)

        if "dns" in data:
            if "ips" in data["dns"]:
                self.assertTrue(True)
            else:
                self.fail("The IPs must be on the data")
        else:
            self.fail("The DNS must be on the data")

    def test18(self):
        """Check integrity of banned domains on DNS traffic."""

        def domain_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        dom = pyaiengine.DomainName("Google test", ".google.com")

        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        # we are not interested on traffic from google
        self.stack.set_domain_name_manager(dman, "dns", False)
        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        # There is no callbacks for domains that we are not
        # interested
        self.assertEqual(self.called_callback, 0)
        self.assertEqual(dom.matchs, 1)
        data = self.stack.get_cache_data("DNSProtocol", "name")

        flow = [flow for flow in self.stack.udp_flow_manager][0]

        self.assertEqual("www.google.com" in data, False)
        self.assertNotEqual(flow, None)
        self.assertNotEqual(flow.dns_info, None)
        self.assertEqual(flow.dns_info.domain_name, "")

    def test19(self):
        """Check the operator * on the DomainName."""

        def domain_callback(flow):
            self.assertIsNotNone(flow)
            self.called_callback += 1

        dom = pyaiengine.DomainName("All domains", "*")

        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)
        self.stack.set_domain_name_manager(dman, "dns")

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        self.assertEqual(self.called_callback, 1)
        self.assertEqual(dom.matchs, 1)
        data = self.stack.get_cache_data("dns", "name")

        flow = [flow for flow in self.stack.udp_flow_manager][0]

        self.assertEqual("www.google.com" in data, True)
        self.assertNotEqual(flow, None)
        self.assertNotEqual(flow.dns_info, None)
        self.assertEqual(flow.dns_info.domain_name, "www.google.com")

    def test20(self):
        """Disable DNS traffic."""

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        counters = self.stack.get_counters("dns")
        counters1 = {"type MX": 0, "type DS": 0, "type SOA": 0, "type CNAME": 0,
                     "responses": 1, "type SRV": 0, "type TXT": 0, "type ANY": 0,
                     "type others": 0, "type SSHFP": 0, "type LOC": 0, "type DNSKEY": 0,
                     "type IXFR": 0, "type AAAA": 1, "type NS": 0, "queries": 1,
                     "allow queries": 1, "banned queries": 0, "type PTR": 0, "type A": 0,
                     "bytes": 92,  "packets": 2, "type HTTPS": 0, "type TLSA": 0,
                     "rcode noerror": 1, "rcode formerr": 0, "rcode servfail": 0,
                     "rcode nxdomain": 0, "rcode notimp": 0, "rcode refused": 0,
                     "rcode yxdomain": 0, "rcode xrrset": 0, "rcode notauth": 0,
                     "rcode notzone": 0, "rcode others": 0}

        self.assertDictEqual(counters, counters1)

        counters2 = {"bytes": 0, "packets": 0}
        counters = self.stack.get_counters("UDPGeneric")

        self.assertDictEqual(counters, counters2)

        self.stack.udp_flow_manager.flush()
        self.stack.disable_protocol("dns")

        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        counters = self.stack.get_counters("DNS")

        self.assertDictEqual(counters, counters1)

        counters3 = {"bytes": 92, "packets": 2}
        counters = self.stack.get_counters("udpgeneric")

        self.assertDictEqual(counters, counters3)

    def test21(self):
        """Disable and enable HTTP."""

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("httP")
        counters1 = {"L7 bytes": 394393, "heads": 0, "responses": 11, "puts": 0, "packets": 318,
                     "bytes": 400490, "connects": 0, "options": 0, "posts": 0, "banned hosts": 0,
                     "others": 0, "requests": 11, "gets": 11, "traces": 0, "allow hosts": 11,
                     "deletes": 0}

        self.assertDictEqual(counters, counters1)

        counters2 = {"bytes": 0, "packets": 0}
        counters = self.stack.get_counters("tcpgeneric")

        self.assertDictEqual(counters, counters2)

        self.stack.tcp_flow_manager.flush()
        self.stack.disable_protocol("http")

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("httP")
        self.assertDictEqual(counters, counters1)

        counters3 = {"bytes": 400490, "packets": 318}
        counters = self.stack.get_counters("tcpgeneric")

        self.assertDictEqual(counters, counters3)

    def test22(self):
        """DHCPv6 traffic test."""

        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/ipv6_dhcp6.pcap")

        data_cache = self.stack.get_cache_data("dhcpv6", "host")
        cache1 = {"TSE-MANAGEMENT": 1}
        self.assertDictEqual(data_cache, cache1)

        for flow in self.stack.udp_flow_manager:
            _ = flow.dhcp6_info.host_name
            _ = flow.dhcp6_info.ip

    def test23(self):
        """Verify the retrieval of the internal caches on DHCPv6."""

        self.stack.decrease_allocated_memory("dhcpv6", 1000)

        cache_host = self.stack.get_cache("dhcpv6", "host")
        self.assertIsNotNone(cache_host)

        cache_ip = self.stack.get_cache("dhcpv6", "ip")
        self.assertIsNotNone(cache_ip)

        self.stack.set_dynamic_allocated_memory(True)

        cache_host.dynamic_allocated_memory = False
        cache_ip.dynamic_allocated_memory = False

        self.inject("../pcapfiles/ipv6_dhcp6.pcap")

        for flow in self.stack.udp_flow_manager:
            _ = flow.dhcp6_info.host_name
            _ = flow.dhcp6_info.ip
            self.assertEqual(flow.dhcp6_info.host_name, "")
            self.assertEqual(flow.dhcp6_info.ip, "")

    def test24(self):
        """Verify the functionality get the internal http caches."""

        def callback_domain(flow):
            self.called_callback += 1
            self.assertIsNotNone(flow)

        dom = pyaiengine.DomainName("Music domain", ".us.listen.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTPProtocol")

        cache_host = self.stack.get_cache("http", "host")
        self.assertIsNotNone(cache_host)
        cache_host.destroy(cache_host.total_items)

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        self.assertEqual(self.called_callback, 0)
        self.assertEqual(dom.matchs, 0)

    def test25(self):
        """Verify DTLS traffic use cases."""

        self.stack.set_dynamic_allocated_memory(True)

        self.inject("../pcapfiles/ipv6_dtls.pcap")

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.dtls_info.pdus, 2)

        counters = self.stack.get_counters("dtls")

        self.assertEqual(counters["records"], 16)
        self.assertEqual(counters["server hellos"], 1)
        self.assertEqual(counters["packets"], 16)
        self.assertEqual(counters["datas"], 2)

        self.stack.reset_counters("dtls")

        counters = self.stack.get_counters("dtls")

        self.assertEqual(counters["records"], 0)
        self.assertEqual(counters["server hellos"], 0)
        self.assertEqual(counters["packets"], 0)
        self.assertEqual(counters["datas"], 0)

    def test26(self):
        """Verify the detach functionality of the Flow."""

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        self.assertEqual(len(self.stack.tcp_flow_manager), 1)

        cache = self.stack.get_cache("http", "host")
        self.assertEqual(cache.total_acquires, 1)
        self.assertEqual(cache.total_fails, 0)
        self.assertEqual(cache.total_releases, 0)

        for flow in self.stack.tcp_flow_manager:
            self.assertIsNotNone(flow.http_info)
            self.assertEqual(flow.l7protocol, "HTTP")
            self.assertEqual(flow.http_info.host_name, "media.us.listen.com")

            # Detach the flow from the HTTPProtocol 
            flow.detach()

            self.assertIsNone(flow.http_info)
            self.assertEqual(flow.l7protocol, "None")

        # There is no changes on the releases because the host
        # could be shared with other flows
        self.assertEqual(cache.total_acquires, 1)
        self.assertEqual(cache.total_fails, 0)
        self.assertEqual(cache.total_releases, 0)

    def test27(self):
        """Verify classification with stack mode changes."""

        self.stack.mode = "nids"

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("HTTP")
        self.assertEqual(counters["packets"], 0)

        counters = self.stack.get_counters("TCPGENERIC")
        self.assertEqual(counters["packets"], 318)

        counters = self.stack.get_counters("TCPFrequencY")
        self.assertEqual(counters["packets"], 0)

        self.stack.tcp_flow_manager.flush()

        self.stack.mode = "full"

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("http")
        self.assertEqual(counters["packets"], 318)

        counters = self.stack.get_counters("TCPGENERIC")
        self.assertEqual(counters["packets"], 318)

        counters = self.stack.get_counters("tcpFREQUENCY")
        self.assertEqual(counters["packets"], 0)

        self.stack.tcp_flow_manager.flush()

        self.stack.mode = "frequency"

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        counters = self.stack.get_counters("http")
        self.assertEqual(counters["packets"], 318)

        counters = self.stack.get_counters("TCPGENERIC")
        self.assertEqual(counters["packets"], 318)

        counters = self.stack.get_counters("tcpFREQUENCY")
        self.assertEqual(counters["packets"], 318)

        self.stack.tcp_flow_manager.flush()

    def test28(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [2015:1::64:41205]:6:[2001:db8::124a:80]
        [2014:dead:beef::1:27245]:17:[2001:abcd::1:53]"""

        self.inject("../pcapfiles/http_over_ipv6.pcap")
        self.inject("../pcapfiles/ipv6_google_dns.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            data = file.read()
            self.assertIn(b"[2015:1::64:41205]:6:[2001:db8::124a:80]", data)
            self.assertIn(b"[2014:dead:beef::1:27245]:17:[2001:abcd::1:53", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="2001:db8::124a")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[2015:1::64:41205]:6:[2001:db8::124a:80]", data)
            self.assertNotIn(b"[2014:dead:beef::1:27245]:17:[2001:abcd::1:53", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(src_ip="2014:dead:beef::1")
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[2015:1::64:41205]:6:[2001:db8::124a:80]", data)
            self.assertIn(b"[2014:dead:beef::1:27245]:17:[2001:abcd::1:53", data)

    def test29(self):
        """Test the learner and frequencies with all flows
        and use the generated regex expression."""

        learner = pyaiengine.LearnerEngine()
        freq_udp = pyaiengine.FrequencyGroup()

        self.stack.mode = "frequency"

        self.inject("../pcapfiles/ipv6_mix_traffic.pcap")

        # Add all the UDP Flows of the FlowManager on the FrequencyEngine
        freq_udp.add_all_flows(self.stack.udp_flow_manager)
        freq_udp.compute()

        self.assertEqual(freq_udp.total_computed_frequencies, 1)
        self.assertEqual(freq_udp.total_process_flows, 16)

        flow_list = freq_udp.get_reference_flows()
        learner.agregate_flows(flow_list)
        learner.compute()

        # Now we use the generated regular expression
        self.stack.mode = "nids"
        self.stack.udp_flow_manager.flush()

        rman = pyaiengine.RegexManager()
        short_regex = "^.{2}\\x00\\x00\\xf9\\xc8\\xe7\\x36"
        reg = pyaiengine.Regex("automatic regex", short_regex)

        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman
        
        self.assertIn(short_regex, learner.regex)
        self.inject("../pcapfiles/ipv6_mix_traffic.pcap")

        self.assertEqual(reg.matchs, 12)

    def test30(self):
        """ Verify UDP Database Adaptors on the IPv6 stack."""

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/ipv6_dtls.pcap")

        data = json.loads(adaptor.lastdata)
        self.assertEqual(data["dtls"]["pdus"], 0)
        self.assertEqual(data["ip"]["dst"], "2a03:39a0:1f:1004:b93c:3e15:d1e3:6848")
        self.assertEqual(data["ip"]["src"], "2a03:39a0:1f:1000:38b6:67b7:3eea:fe28")
        self.assertEqual(data["layer7"], "DTLS")

    def test31(self):
        """Verify the metrics of packets and bytes up an down stream."""

        self.inject("../pcapfiles/ipv6_bgp.pcapng")

        flow = [flow for flow in self.stack.tcp_flow_manager][0]

        self.assertEqual(flow.upstream_ttl, 255)
        self.assertEqual(flow.downstream_ttl, 255)
        self.assertEqual(flow.bytes, flow.upstream_bytes + flow.downstream_bytes)
        self.assertEqual(flow.upstream_packets, 21)
        self.assertEqual(flow.downstream_packets, 20)
        self.assertEqual(flow.packets, flow.upstream_packets + flow.downstream_packets)
        self.assertEqual(flow.duration, 205)

    def test32(self):
        """Verify that changes on pcap file formats are process correctly."""

        self.pdis.open("../pcapfiles/ipv6_dtls.pcap")
        self.pdis.run()
        self.assertEqual(self.pdis.packets, 16)
        self.assertEqual(self.pdis.pcap_files_duration, 6)

        counters = self.stack.get_counters("udp")
        self.assertEqual(counters["packets"], 16)

        self.pdis.open("../pcapfiles/ipv6_bgp.pcapng")
        self.pdis.run()
        self.assertEqual(self.pdis.packets, 41 + 16)
        self.assertEqual(self.pdis.pcap_files_duration, 6 + 205)

        counters = self.stack.get_counters("tcp")
        self.assertEqual(counters["packets"], 41)

        self.stack.udp_flow_manager.flush()

        self.pdis.open("../pcapfiles/ipv6_dtls.pcap")
        self.pdis.run()
        self.assertEqual(self.pdis.packets, 16 + 41 + 16)
        self.assertEqual(self.pdis.pcap_files_duration, 6 + 205 + 6)

        counters = self.stack.get_counters("udp")
        self.assertEqual(counters["packets"], 16 + 16)

    def test33(self):
        """Verify the data on the HTTP cache."""

        self.pdis.open("../pcapfiles/http_over_ipv6.pcap")
        self.pdis.run()
        self.pdis.close()

        data = self.stack.get_cache_data("HTTP", "Uri")

        self.assertIn("/listenmedia/albumimage/01/02/72/I2JLEN2L67H8.png", data)
        self.assertEqual(data["/listenmedia/albumimage/01/02/72/I2JLEN2L67H8.png"], 3)

        self.assertIn("/listenmedia/albumimage/01/02/72/AK0NFRKVVYG8.png", data)
        self.assertEqual(data["/listenmedia/albumimage/01/02/72/AK0NFRKVVYG8.png"], 1)

        self.stack.release_cache("HTTP")
        data = self.stack.get_cache_data("HTTP", "Uri")
        self.assertFalse(data)

    def test34(self):
        """Verify the * on the HTTPUriSet."""

        def callback_uri(flow):
            self.called_callback += 1

        dom = pyaiengine.DomainName("Music domain", ".us.listen.com")
 
        uset = pyaiengine.HTTPUriSet("All the uris")

        # Any uri will match
        uset.add_uri("*")
        uset.callback = callback_uri

        dman = pyaiengine.DomainNameManager()
        dom.http_uri_set = uset
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "HTTP")

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        self.assertEqual(self.called_callback, 11)

        # Verify the output of the HTTPUriSet 
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            uset.show()
            file.seek(0)
            total_lines = len(file.readlines())
            self.assertEqual(total_lines, 2)

    def test35(self):
        """Verify the functionality when there is a cache fail."""

        self.called_callback = 0

        def fail_cache_callback(flow):
            self.called_callback += 1
            self.assertEqual("2015:1::64:41205:6:2001:db8::124a:80", str(flow))

        # Reduce the memory for store HTTP information
        self.stack.decrease_allocated_memory("HTTP", 2000)

        # Set the callback for be called on a cache fail
        self.stack.set_on_fail_cache_callback("HTTP", fail_cache_callback)

        self.inject("../pcapfiles/http_over_ipv6.pcap")

        self.assertEqual(self.called_callback , 1)

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test36(self):
        """Create some IPs on an IPSet and verify that the alerted
        flow generates the alert."""

        def ipset_callback(flow):
            data = {"key": "value"}
            flow.label = json.dumps(data)
            flow.alerted = True
            self.called_callback += 1

        items = []
        for i in range(1, 100):
            items.append(f"2003:de:2016:110::b15:{i}")

        ipset = pyaiengine.IPSet("IPv6 generic set", items)
        ipset.callback = ipset_callback
        im = pyaiengine.IPSetManager()

        self.assertEqual(len(ipset), len(items))

        im.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = im

        self.stack.link_layer_tag = "vlan"
        self.stack.mode = "nids"

        self.inject("../pcapfiles/ipv6_ssh.pcap")

        self.assertEqual(self.called_callback, 1)

        # Now we sleep and reads from the queue to see message
        # generated by the alerted property
        time.sleep(QUEUE_DELAY)
        ret = subprocess.run(["./alarmreader", "-t", "1"], stdout=subprocess.PIPE)
        lines = ret.stdout.decode("utf-8").split("\n")
        self.assertIn("StackLanIPv6Tests.test36.<locals>.ipset_callback", lines[0])
        try:
            data = json.loads("\n".join(lines[1:]))
            self.assertEqual(data["ip"]["dst"], "2003:de:2016:110::b15:22")
            self.assertEqual(data["ipset"], ipset.name)
            self.assertEqual(data["label"], "{\"key\": \"value\"}")
        except:
            self.assertTrue(False)


class StackLanLearningTests(unittest.TestCase):

    def setUp(self):
        self.stack = pyaiengine.StackLan()
        self.pdis = pyaiengine.PacketDispatcher()
        self.pdis.stack = self.stack
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.freq = pyaiengine.FrequencyGroup()

    def tearDown(self):
        pass

    def inject(self, pcapfile, pcapfilter=None):
        self.pdis.open(pcapfile)
        if pcapfilter != None:
            self.pdis.pcap_filter = pcapfilter
        self.pdis.run()
        self.pdis.close()

    def test01(self):
        """Sanity test case for verify the frequencies."""

        self.freq.reset()
        self.stack.mode = "frequency"

        self.inject("../pcapfiles/two_http_flows_noending.pcap")

        self.assertEqual(self.freq.total_process_flows, 0)
        self.assertEqual(self.freq.total_computed_frequencies, 0)

        # Add the TCP Flows of the FlowManager on the FrequencyEngine
        self.freq.add_flows_by_destination_port(self.stack.tcp_flow_manager)
        self.freq.compute()

        self.assertEqual(self.freq.total_process_flows, 2)
        self.assertEqual(self.freq.total_computed_frequencies, 1)

    def test02(self):

        self.freq.reset()
        self.stack.mode = "frequency"

        self.inject("../pcapfiles/tor_4flows.pcap")

        self.assertEqual(self.freq.total_process_flows, 0)
        self.assertEqual(self.freq.total_computed_frequencies, 0)

        # Add the TCP Flows of the FlowManager on the FrequencyEngine
        self.freq.add_flows_by_destination_port(self.stack.tcp_flow_manager)
        self.freq.compute()

        self.assertEqual(len(self.freq.get_reference_flows_by_key("80")), 4)
        self.assertEqual(len(self.freq.get_reference_flows()), 4)
        self.assertEqual(len(self.freq.get_reference_flows_by_key("8080")), 0)
        self.assertEqual(self.freq.total_process_flows, 4)
        self.assertEqual(self.freq.total_computed_frequencies, 1)

    def test03(self):
        """Integrate with the learner to generate a regex."""
        learn = pyaiengine.LearnerEngine()

        self.freq.reset()
        self.stack.mode = "frequency"

        self.inject("../pcapfiles/tor_4flows.pcap")

        # Add the TCP Flows of the FlowManager on the FrequencyEngine
        self.freq.add_flows_by_destination_port(self.stack.tcp_flow_manager)
        self.freq.compute()

        flow_list = self.freq.get_reference_flows()
        self.assertEqual(self.freq.total_computed_frequencies, 1)
        learn.agregate_flows(flow_list)
        learn.compute()

        self.assertEqual(learn.flows_process, 4)

        # Get the generated regex and compile with the regex module
        try:
            _ = re.compile(learn.regex)
            self.assertTrue(True)
        except:
            self.assertFalse(False)

    def test04(self):
        """ Switch from normal mode to learner mode and check flow and caches status """
        learn = pyaiengine.LearnerEngine()

        self.freq.reset()
        self.stack.mode = "frequency"

        # The filter tcp and port 55354 will filter just one HTTP flow
        # that contains exactly 39 requests and 38 responses
        self.inject("../pcapfiles/two_http_flows_noending.pcap", pcapfilter="tcp and port 55354")

        # Add the TCP Flows of the FlowManager on the FrequencyEngine
        self.freq.add_flows_by_destination_port(self.stack.tcp_flow_manager)
        self.freq.compute()

        flow_list = self.freq.get_reference_flows()
        self.assertEqual(self.freq.total_computed_frequencies, 1)
        learn.agregate_flows(flow_list)
        learn.compute()

        self.assertEqual(learn.flows_process, 1)
        self.assertEqual(len(self.stack.tcp_flow_manager), 1)

        flow1 = [flow for flow in self.stack.tcp_flow_manager][0]

        # Switch to normal mode and inject the other flow

        self.stack.mode = "full"
        self.inject("../pcapfiles/two_http_flows_noending.pcap", pcapfilter="tcp and port 49503")

        flow2 = [flow for flow in self.stack.tcp_flow_manager if flow.src_port == 49503][0]

        self.assertIsNotNone(flow1)
        self.assertEqual(flow1.l7protocol, "TCPFrequency")
        self.assertIsNotNone(flow1.frequencies)
        self.assertIsNotNone(flow1.packet_frequencies)
        self.assertIsNotNone(flow2)
        self.assertEqual(flow2.l7protocol, "HTTP")
        self.assertIsNotNone(flow2.http_info)
        self.assertIsNone(flow2.frequencies)
        self.assertEqual(len(self.stack.tcp_flow_manager), 2)

        self.stack.release_caches()

        self.assertEqual(flow1.l7protocol, "TCPFrequency")
        self.assertIsNone(flow1.frequencies)
        self.assertIsNone(flow1.packet_frequencies)
        self.assertEqual(flow2.l7protocol, "HTTP")
        self.assertIsNone(flow2.http_info)
        self.assertIsNone(flow2.frequencies)


class StackVirtualTests(unittest.TestCase):
    """Handle the test cases for the VirtualStacks."""

    def setUp(self):
        self.stack = pyaiengine.StackVirtual()
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile):
        """Process the pcapfile."""

        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """Create a regex for a detect the flow on a virtual network."""

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("Bin directory", "^bin$")
        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/vxlan_ftp.pcap")

        self.assertEqual(reg.matchs, 1)

    def test02(self):
        """Create a regex for a detect the flow on a virtual network on the GRE."""

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("Bin directory", b"^SSH-2.0.*$")
        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/gre_ssh.pcap")

        self.assertEqual(reg.matchs, 1)

        self.assertEqual(len(self.stack.tcp_flow_manager), 1)
        self.assertEqual(len(self.stack.udp_flow_manager), 0)

    def test03(self):
        """Inject two pcapfiles with gre and vxlan traffic and verify regex."""

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("SSH activity", b"^SSH-2.0.*$")
        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        self.stack.mode = "nids"

        # The first packet of the pcapfile is from 18 sep 2014
        self.inject("../pcapfiles/vxlan_ftp.pcap")

        # This FlowManagers points to the virtualize layer
        ft = self.stack.tcp_flow_manager
        fu = self.stack.udp_flow_manager

        self.assertEqual(ft.flows, 1)
        self.assertEqual(ft.process_flows, 1)
        self.assertEqual(ft.timeout_flows, 0)

        self.assertEqual(reg.matchs, 0)
        self.assertEqual(len(self.stack.tcp_flow_manager), 1)
        self.assertEqual(len(self.stack.udp_flow_manager), 0)

        self.stack.flows_timeout = (60 * 60 * 24)

        # The first packet of the pcapfile is from 19 sep 2014
        self.inject("../pcapfiles/gre_ssh.pcap")

        self.assertEqual(ft.flows, 2)
        self.assertEqual(ft.process_flows, 2)
        self.assertEqual(ft.timeout_flows, 0)

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(len(ft), 2)
        self.assertEqual(len(fu), 0)

    def test04(self):
        """Test the extraction of the tag from the flow when matches."""

        def virt_callback(flow):
            if (flow.have_tag)and(flow.tag == 1):
                self.called_callback += 1

        def other_callback(_):
            pass

        # Some callback reference verifications  
        self.assertEqual(sys.getrefcount(virt_callback), 2)
        self.assertEqual(sys.getrefcount(other_callback), 2)
        reg = pyaiengine.Regex("Bin directory", b"^bin$")
        reg.callback = virt_callback
        self.assertEqual(sys.getrefcount(virt_callback), 3)
        self.assertEqual(sys.getrefcount(other_callback), 2)

        reg.callback = other_callback
        self.assertEqual(sys.getrefcount(virt_callback), 2)
        self.assertEqual(sys.getrefcount(other_callback), 3)

        reg.callback = None
        self.assertEqual(sys.getrefcount(virt_callback), 2)
        self.assertEqual(sys.getrefcount(other_callback), 2)

        reg.callback = virt_callback
        self.assertEqual(sys.getrefcount(virt_callback), 3)
        rman = pyaiengine.RegexManager()
        rman.add_regex(reg)
        self.assertEqual(sys.getrefcount(virt_callback), 3)
        self.stack.tcp_regex_manager = rman

        self.stack.mode = "nids"

        self.inject("../pcapfiles/vxlan_ftp.pcap")

        self.assertEqual(reg.callback, virt_callback)
        self.assertEqual(reg.matchs, 1)
        self.assertEqual(self.called_callback, 1)

        reg.callback = None
        self.assertEqual(sys.getrefcount(virt_callback), 2)

    def test05(self):
        """Verify regex on the constructor for easy management."""

        # Create a multi regex that match with different packets
        reg = pyaiengine.Regex("First that matchs", b"^SSH-2.0.*$", \
            pyaiengine.Regex("Second that matchs", b"^SSH-2.0.*$", \
            pyaiengine.Regex("Third that matchs", b"^.*diffie-hellman.*$", \
            pyaiengine.Regex("For dont that matchs", b"This can not match"))))

        rman = pyaiengine.RegexManager()
        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman
        self.stack.mode = "nids"

        self.inject("../pcapfiles/gre_ssh.pcap")

        reg1 = reg.next_regex
        reg2 = reg.next_regex.next_regex
        reg3 = reg.next_regex.next_regex.next_regex

        self.assertEqual(reg.matchs, 1)
        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg3.matchs, 0)

        info = [flow.ssh_info for flow in self.stack.tcp_flow_manager][0]

        self.assertEqual(info, None)

    def test06(self):
        """Verify regex on the constructor for easy management."""

        # Create a regex list that match with different packets
        rman = pyaiengine.RegexManager("Some regexs", \
              [pyaiengine.Regex("First that matchs", b"^SSH-2.0.*$"), \
              pyaiengine.Regex("Second dont matchs", b"^SSH-2.0.*$"), \
              pyaiengine.Regex("Third dont matchs", b"^.*diffie-hellman.*$"), \
              pyaiengine.Regex("Four dont that matchs", b"This can not match")])

        self.stack.tcp_regex_manager = rman
        self.stack.mode = "nids"

        self.inject("../pcapfiles/gre_ssh.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
             # Just match the first regex
            rman.show()
            file.seek(0)
            lines = file.readlines()
            self.assertIn(b"Name:First that matchs         Matchs:1", lines[1])
            self.assertIn(b"Name:Second dont matchs        Matchs:0", lines[2])

    def test07(self):
        """Enable and Disable the DHCP protocol."""

        self.inject("../pcapfiles/gre_dhcp.pcap")

        data_cache = self.stack.get_cache_data("dhcp", "host")
        data1 = {"PAQUITO": 1}
        self.assertDictEqual(data_cache, data1)

        for flow in self.stack.udp_flow_manager:
            _ = flow.dhcp_info.host_name
            _ = flow.dhcp_info.ip

        counters1 = {"releases": 0, "packets": 1, "bytes": 253, "informs": 0, "offers": 0,
                     "discovers": 1, "acks": 0, "declines": 0, "requests": 0, "naks": 0}
        counters2 = {"bytes": 0, "packets": 0}
        counters = self.stack.get_counters("dhcp")

        self.assertDictEqual(counters, counters1)

        counters = self.stack.get_counters("udpgeneric")
        self.assertDictEqual(counters, counters2)

        self.stack.udp_flow_manager.flush()
        self.stack.disable_protocol("dhcp")

        self.inject("../pcapfiles/gre_dhcp.pcap")

        counters = self.stack.get_counters("dhcp")

        self.assertDictEqual(counters, counters1)

        counters3 = {"bytes": 253, "packets": 1}
        counters = self.stack.get_counters("udpgeneric")

        self.assertDictEqual(counters, counters3)

    def test08(self):
        """Verify the retrieval of the internal caches on DHCP."""

        cache = self.stack.get_cache("dhcp", "name")
        cache.destroy(cache.total_items)

        self.inject("../pcapfiles/gre_dhcp.pcap")

        cache = self.stack.get_cache_data("dhcp", "nothing")
        cache1 = {}
        self.assertDictEqual(cache, cache1)

    def test09(self):
        """Verify the detach functionality of the Flow."""

        self.inject("../pcapfiles/gre_dhcp.pcap")

        cache = self.stack.get_cache("dhcp", "host")

        self.assertEqual(cache.total_acquires, 1)
        self.assertEqual(cache.total_fails, 0)
        self.assertEqual(cache.total_releases, 0)

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.l7protocol, "DHCP") 
            self.assertIsNotNone(flow.dhcp_info)
            self.assertEqual(flow.dhcp_info.host_name, "PAQUITO")        
   
            # Detach the flow from the protocol
            flow.detach()
 
            self.assertEqual(flow.l7protocol, "None") 
            self.assertIsNone(flow.dhcp_info)

        # The flow detach the DHCPInfo but the host
        # stil not been release to the cache
        self.assertEqual(cache.total_acquires, 1)
        self.assertEqual(cache.total_fails, 0)
        self.assertEqual(cache.total_releases, 0)

    def test10(self):
        """Verify changes on the mode of operation works
        correctly with no issues."""

        self.stack.mode = "frequency"

        self.inject("../pcapfiles/gre_dhcp.pcap")

        counters = self.stack.get_counters("GRE")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("DHCP")
        self.assertEqual(counters["packets"], 0)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 0)

        # Remnove the flows
        self.stack.udp_flow_manager.flush()

        # Now change the mode of operation
        self.stack.mode = "full"

        self.inject("../pcapfiles/gre_dhcp.pcap")

        counters = self.stack.get_counters("GRE")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("DHCP")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 0)

        # Remnove the flows
        self.stack.udp_flow_manager.flush()

        # Now change the mode of operation
        self.stack.mode = "nids"

        self.inject("../pcapfiles/gre_dhcp.pcap")

        counters = self.stack.get_counters("GRE")
        self.assertEqual(counters["packets"], 3)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("DHCP")
        self.assertEqual(counters["packets"], 1)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 1)

    def test11(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [192.168.1.2:41024]:17:[192.168.1.1:4789] vxlan
        [192.168.1.23:20]:6:[192.168.1.100:1034] tcpgeneric
        [192.168.0.1:67]:17:[255.255.255.255:67] dhcp""" 

        self.inject("../pcapfiles/gre_dhcp.pcap")
        self.inject("../pcapfiles/vxlan_ftp.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(protocol=6)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertNotIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="192.168.1.1")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertNotIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertNotIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(src_ip="192.168.1.23", dst_ip="192.168.1.100")
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertNotIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="192.168.1.0/24")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertNotIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(src_ip="192.168.1.0/24", dst_ip="192.168.1.100/24")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.1.2:41024]:17:[192.168.1.1:4789]", data)
            self.assertIn(b"[192.168.1.23:20]:6:[192.168.1.100:1034]", data)
            self.assertNotIn(b"[192.168.0.1:67]:17:[255.255.255.255:67]", data)

    def test12(self):
        """Verify the functionality when there is a cache fail."""

        self.called_callback = 0

        def fail_cache_callback(flow):
            self.called_callback += 1
            self.assertEqual("192.168.0.1:67:17:255.255.255.255:67", str(flow))

        # Reduce the memory for store DHCP information
        self.stack.decrease_allocated_memory("DHCP", 1000)

        # Set the callback for be called on a cache fail
        self.stack.set_on_fail_cache_callback("DHCP", fail_cache_callback)

        self.inject("../pcapfiles/gre_dhcp.pcap")

        self.assertEqual(self.called_callback , 1)


class StackOpenFlowTests(unittest.TestCase):
    """This class handlers test cases related to OpenFlow traffic."""

    def setUp(self):
        self.stack = pyaiengine.StackOpenFlow()
        self.pdis = pyaiengine.PacketDispatcher()
        self.pdis.stack = self.stack
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile, pcapfilter=""):
        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            if pcapfilter:
                pdis.pcap_filter = pcapfilter
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """ Create a regex for a detect the flow on a openflow network """

        rman = pyaiengine.RegexManager()
        reg1 = pyaiengine.Regex("Bin directory", b"^\\x26\\x01")
        reg2 = pyaiengine.Regex("All", b"^.*$")
        rman.add_regex(reg1)
        rman.add_regex(reg2)
        self.stack.tcp_regex_manager = rman

        self.inject("../pcapfiles/openflow.pcap")

        self.assertEqual(reg1.matchs, 0)
        self.assertEqual(reg2.matchs, 1)

    def test02(self):
        """Test the with statement of the PacketDispatcher."""

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("Bin directory", b"^\\x26\\x01")

        # We want to see the matched packet
        reg.write_packet = True

        rman.add_regex(reg)
        self.stack.tcp_regex_manager = rman

        adaptor = DatabaseTestAdaptor()
        self.stack.set_tcp_database_adaptor(adaptor, 1)

        self.inject("../pcapfiles/openflow.pcap")

        data = json.loads(adaptor.lastdata)

        if "matchs" in data:
            self.assertEqual(data["matchs"], "Bin directory")
        else:
            self.assertTrue(False)

        self.assertEqual(reg.matchs, 1)

        # the packet is write on the packet field of the json
        self.assertIn("l7_payload", data)
        packet = data["l7_payload"]

        self.assertEqual(packet[0], 38)
        self.assertEqual(packet[1], 1)
        self.assertEqual(reg.write_packet, True)

    def test03(self):
        """Test the flowmanager flush functionality."""

        self.inject("../pcapfiles/openflow.pcap")

        tcp_flows = self.stack.tcp_flow_manager

        self.assertEqual(tcp_flows.flows, 1)
        self.assertEqual(tcp_flows.process_flows, 1)
        self.assertEqual(tcp_flows.timeout_flows, 0)
        self.assertEqual(len(tcp_flows), 1)

        tcp_flows.flush()

        self.assertEqual(tcp_flows.flows, 0)
        self.assertEqual(tcp_flows.process_flows, 1)
        self.assertEqual(tcp_flows.timeout_flows, 0)
        self.assertEqual(len(tcp_flows), 0)

    def test04(self):
        """Verify DNS query on openflow, be aware that the pcapfile
           contains one openflow flow and two dns, one query and one
           response that are different each other."""

        def domain_callback(flow):
            self.assertNotEqual(flow.dns_info, None)
            self.assertEqual(flow.dns_info.domain_name, "daisy.ubuntu.com")
            self.called_callback += 1
            self.assertEqual(flow.upstream_ttl, 62)
            self.assertEqual(flow.downstream_ttl, 0)

        dom = pyaiengine.DomainName("test", ".ubuntu.com")

        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        adaptor = DatabaseTestAdaptor()
        self.stack.set_udp_database_adaptor(adaptor, 1)

        self.stack.set_domain_name_manager(dman, "dns")

        self.inject("../pcapfiles/openflow_dns.pcap")

        cache = self.stack.get_cache_data("DNS", "name")
        cache1 = {"daisy.ubuntu.com": 2}

        self.assertDictEqual(cache, cache1)
        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)

        # Verify the output of adaptor
        data = json.loads(adaptor.lastdata)

        data1 = {u"dns": {u"domain": u"daisy.ubuntu.com",
                          u"ips": [u"91.189.92.55", u"91.189.92.57"],
                          u"matchs": u"test", u"qtype": 0}, 
                 u"layer7": u"DNS", u"packets": 1, u"proto": 17,
                 u"ip": {u"src": u"129.21.3.17", u"dst": u"192.168.2.6"},
                 u"evidence": False,
                 u"reject": False,
                 u"upstream_ttl":62,
                 u"downstream_ttl":0,
                 u"bytes": 94, u"anomaly": 6, u"port": {u"src": 53, u"dst": 28848}}

        if defined("HAVE_REJECT_FLOW"):
            data1["reject"] = False
        # print(json.dumps(data, sort_keys=True, indent=4, separators=(",", ": ")))
        self.assertEqual(data["dns"]["domain"], data1["dns"]["domain"])

    def test05(self):
        """Enable and disable DNS protocol."""

        self.inject("../pcapfiles/openflow_dns.pcap")

        c1 = {"type MX": 0, "type DS": 0, "type SOA": 0, "type CNAME": 0, "responses": 1,
              "type SRV": 0, "type TXT": 0, "type ANY": 0, "type others": 0, "type SSHFP": 0,
              "type LOC": 0, "type DNSKEY": 0, "type IXFR": 0, "type AAAA": 1, "type NS": 0,
              "queries": 1, "allow queries": 1, "banned queries": 0, "type PTR": 0, "type A": 1,
              "packets": 2, "bytes": 120, "type HTTPS": 0, "type TLSA": 0,
              "rcode noerror": 1, "rcode formerr": 0, "rcode servfail": 0, "rcode nxdomain": 0,
              "rcode notimp": 0, "rcode refused": 0, "rcode yxdomain": 0, "rcode xrrset": 0,
              "rcode notauth": 0, "rcode notzone": 0, "rcode others": 0}

        c2 = {"bytes": 0, "packets": 0}
        counters = self.stack.get_counters("DNS")

        self.assertDictEqual(counters, c1)

        counters = self.stack.get_counters("udpgeneric")

        self.assertDictEqual(counters, c2)

        self.stack.udp_flow_manager.flush()

        self.stack.disable_protocol("dns")
        self.inject("../pcapfiles/openflow_dns.pcap")

        counters = self.stack.get_counters("DNS")

        self.assertDictEqual(counters, c1)

        c3 = {"bytes": 120, "packets": 2}
        counters = self.stack.get_counters("UdPgEnErIc")

        self.assertDictEqual(counters, c3)

        # Test the output of show_anomalies and show_protocol_statistics
        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_anomalies()
            self.stack.show_protocol_statistics()

    def test06(self):
        """Verify changes on the mode of operation works
        correctly with no issues."""

        self.stack.mode = "frequency"

        self.inject("../pcapfiles/openflow.pcap")

        counters = self.stack.get_counters("openflow")
        self.assertEqual(counters["packets"], 18)

        counters = self.stack.get_counters("TCPFREQUENCY")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("TCPGENERIC")
        self.assertEqual(counters["packets"], 0)

        # Remnove the flows
        self.stack.tcp_flow_manager.flush()

        # Now change the mode of operation
        self.stack.mode = "full"

        self.inject("../pcapfiles/openflow.pcap")

        counters = self.stack.get_counters("openflow")
        self.assertEqual(counters["packets"], 18 + 18)

        counters = self.stack.get_counters("TCPFREQUENCY")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("TCPGENERIC")
        self.assertEqual(counters["packets"], 2)

        # Remnove the flows
        self.stack.tcp_flow_manager.flush()

    def test07(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [192.168.2.4:45203]:6:[192.168.2.14:6633] OpenFlow
        [192.168.2.4:46926]:6:[192.168.2.14:22] tcpgeneric
        [192.168.2.8:1044]:17:[239.255.255.250:8082] udpgeneric"""

        self.inject("../pcapfiles/openflow.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows()
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.2.4:45203]:6:[192.168.2.14:6633]", data)
            self.assertIn(b"[192.168.2.4:46926]:6:[192.168.2.14:22]", data)
            self.assertIn(b"[192.168.2.8:1044]:17:[239.255.255.250:8082]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_port=22)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[192.168.2.4:45203]:6:[192.168.2.14:6633]", data)
            self.assertIn(b"[192.168.2.4:46926]:6:[192.168.2.14:22]", data)
            self.assertNotIn(b"[192.168.2.8:1044]:17:[239.255.255.250:8082]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="192.168.2.14")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.2.4:45203]:6:[192.168.2.14:6633]", data)
            self.assertIn(b"[192.168.2.4:46926]:6:[192.168.2.14:22]", data)
            self.assertNotIn(b"[192.168.2.8:1044]:17:[239.255.255.250:8082]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="192.168.2.14", dst_port=22)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[192.168.2.4:45203]:6:[192.168.2.14:6633]", data)
            self.assertIn(b"[192.168.2.4:46926]:6:[192.168.2.14:22]", data)
            self.assertNotIn(b"[192.168.2.8:1044]:17:[239.255.255.250:8082]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="192.168.2.14", limit=1)
            file.seek(0)
            data = file.read()
            self.assertIn(b"[192.168.2.4:45203]:6:[192.168.2.14:6633]", data)
            self.assertNotIn(b"[192.168.2.4:46926]:6:[192.168.2.14:22]", data)
            self.assertNotIn(b"[192.168.2.8:1044]:17:[239.255.255.250:8082]", data)

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test08(self):
        """Verify that the callback on DNS can generate an alarm
        that will be stored on a system queue."""

        def domain_callback(flow):
            self.assertNotEqual(flow.dns_info, None)
            self.assertEqual(flow.dns_info.domain_name, "daisy.ubuntu.com")
            self.called_callback += 1

            # We set the property alerted of the flow for
            # generate a message on a system queue
            flow.alerted = True

        dom = pyaiengine.DomainName("test", ".ubuntu.com")

        dom.callback = domain_callback

        dman = pyaiengine.DomainNameManager()
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "dns")

        self.inject("../pcapfiles/openflow_dns.pcap")

        self.assertEqual(dom.matchs, 1)
        self.assertEqual(self.called_callback, 1)

        # The message with the flow should be on the queue
        time.sleep(QUEUE_DELAY)
        ret = subprocess.run(["./alarmreader", "-t", "1"], stdout=subprocess.PIPE)
        self.assertEqual(ret.returncode, 0)
        lines = ret.stdout.decode("utf-8").split("\n")
        self.assertIn(b"StackOpenFlowTests.test08.<locals>.domain_callback", ret.stdout)

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test09(self):
        """Create a regex for a detect a flow
        and send an alert to the queue."""

        def regex_callback(flow):
            self.called_callback += 1
            flow.alerted = True

        rman = pyaiengine.RegexManager()
        reg = pyaiengine.Regex("Bin directory", b"NOTIFY")
        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman

        reg.callback = regex_callback

        self.inject("../pcapfiles/openflow.pcap")

        self.assertEqual(reg.matchs, 1)

        # The message with the flow should be on the queue
        time.sleep(QUEUE_DELAY)
        ret = subprocess.run(["./alarmreader", "-t", "1"], stdout=subprocess.PIPE)
        self.assertEqual(ret.returncode, 0)
        lines = ret.stdout.decode("utf-8").split("\n")
        self.assertIn(b"StackOpenFlowTests.test09.<locals>.regex_callback", ret.stdout)


class PacketDispatcherTests(unittest.TestCase):

    def setUp(self):
        self.pdis = pyaiengine.PacketDispatcher()

    def tearDown(self):
        del self.pdis

    def test01(self):
        """The packet dispatcher should process the packets without stack."""

        self.pdis.open("../pcapfiles/vxlan_ftp.pcap")
        self.pdis.run()
        self.pdis.close()
        self.assertEqual(self.pdis.bytes, 900)
        self.assertEqual(self.pdis.packets, 8)

        # Check some default properties
        self.assertEqual(self.pdis.evidences, False)
        self.assertEqual(len(self.pdis.pcap_filter), 0)
        self.assertEqual(self.pdis.is_packet_accepted, True)

        # This variables should be zero
        self.assertEqual(self.pdis.received_packets, 0)
        self.assertEqual(self.pdis.dropped_packets, 0)
        self.assertEqual(self.pdis.ifdropped_packets, 0)

    def test02(self):
        """Check the port functionality."""

        self.pdis.open("../pcapfiles/vxlan_ftp.pcap")
        self.pdis.enable_shell = True

        port = random.randint(2000, 65000)

        self.pdis.port = port
        self.assertEqual(self.pdis.port, port)
        self.assertEqual(self.pdis.enable_shell, True)

        # The socket should be in use
        try:
            # Use psutil for verify if the current process have a socket open on
            # the port.
            import psutil

            proc = psutil.Process(os.getpid())
            conn = proc.connections()[0]
            self.assertEqual(port, conn.laddr[1])
        except:
            pass

        self.pdis.run()
        self.pdis.close()

    def test03(self):
        """Run the dispatcher of a unknown device name."""

        self.pdis.open("I_dont think this will work")
        self.pdis.run()
        self.pdis.close()
        self.assertEqual(self.pdis.bytes, 0)
        self.assertEqual(self.pdis.packets, 0)

    def test04(self):
        """Test case for the add_timer functionality."""

        def timer1():
            pass
        def timer2():
            pass
        def timer3():
            pass

        self.pdis.add_timer(timer1, 1)
        self.pdis.add_timer(timer2, 10)
        self.pdis.add_timer(timer3, 1)
        self.pdis.add_timer(None, 1)

        self.pdis.open("../pcapfiles/vxlan_ftp.pcap")
        self.pdis.run()
        self.pdis.close()

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.pdis.show()

        self.pdis.add_timer(None, 10)

    def test05(self):
        """Check the pcap filter variable"""

        self.pdis.open("../pcapfiles/vxlan_ftp.pcap")
        self.pdis.pcap_filter = "udp"
        self.assertEqual(self.pdis.pcap_filter, "udp")
        self.pdis.run()
        self.pdis.pcap_filter = "this is not a filter"
        self.assertEqual(self.pdis.pcap_filter, "udp")

        self.pdis.close() 

    def test06(self):
        """Verify the authorization_code_token can be set/unset safe."""

        self.pdis.authorization_code_token = "no"
        self.assertEqual(self.pdis.authorization_code_token, "")

        self.pdis.authorization_code_token = "1234567"
        self.assertEqual(self.pdis.authorization_code_token, "")

        self.pdis.authorization_code_token = "12345678"
        self.assertEqual(self.pdis.authorization_code_token, "1234...")

        self.pdis.authorization_code_token = "AAAAAAAAAA"
        self.assertEqual(self.pdis.authorization_code_token, "AAAA...")

    def test07(self):
        """Verify the pcap filters works."""

        self.pdis.open("../pcapfiles/vxlan_ftp.pcap")
        self.pdis.pcap_filter = "tcp port 8888"
        self.assertEqual(self.pdis.pcap_filter, "tcp port 8888")

        self.pdis.pcap_filter = "not working"
        self.assertEqual(self.pdis.pcap_filter, "tcp port 8888")

        self.pdis.pcap_filter = "ip"
        self.assertEqual(self.pdis.pcap_filter, "ip")

        self.pdis.pcap_filter = ""
        self.assertEqual(self.pdis.pcap_filter, "")
        self.pdis.close()

class StackMobileIPv6Tests(unittest.TestCase):
    """Tests related to IPv6 Mobile environments."""

    def setUp(self):
        self.stack = pyaiengine.StackMobileIPv6()
        self.pdis = pyaiengine.PacketDispatcher()
        self.pdis.stack = self.stack
        self.stack.tcp_flows = 2048
        self.stack.udp_flows = 1024
        self.called_callback = 0

    def tearDown(self):
        pass

    def inject(self, pcapfile):
        """Process the pcap file."""

        with pyaiengine.PacketDispatcher(pcapfile) as pdis:
            pdis.stack = self.stack
            pdis.run()

    def test01(self):
        """Verify the integrity of the sip fields."""

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gprs_ip6_tcp.pcap")

        for flow in self.stack.tcp_flow_manager:
            self.assertEqual(flow.mqtt_info, None)
            self.assertEqual(flow.coap_info, None)
            self.assertEqual(flow.http_info, None)
            self.assertEqual(flow.dns_info, None)
            self.assertEqual(flow.ssl_info, None)

        counters1 = {"bytes": 3198, "packets": 11, "fragmented packets": 0,
                     "extension header packets": 0}
        counters = self.stack.get_counters("ipv6")
        self.assertDictEqual(counters, counters1)

    def test02(self):
        """Verify the integrity of the sip fields."""

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gprs_ip6_udp.pcap")

        for flow in self.stack.udp_flow_manager:
            self.assertEqual(flow.mqtt_info, None)
            self.assertEqual(flow.coap_info, None)
            self.assertEqual(flow.http_info, None)
            self.assertEqual(flow.dns_info, None)
            self.assertEqual(flow.ssl_info, None)
            self.assertNotEqual(flow.sip_info, None)
            self.assertEqual(flow.sip_info.from_name, "<tel:+88270006>;tag=9Q5V3XeXXf")
            self.assertEqual(flow.sip_info.to_name, "<tel:+88270006>")
            self.assertEqual(flow.sip_info.uri, "tel:+7")
            self.assertEqual(flow.sip_info.via, \
                "SIP/2.0/UDP [fd00:183:1:1:1886:9040:8605:32b8]:5060;branch=z9hG4bKOJ5umQnnq16M2Cr;rport")

        data = json.loads(adaptor.lastdata)
        self.assertEqual(data["ip"]["dst"], "fd01::183")
        self.assertEqual(data["ip"]["src"], "fd00:183:1:1:1886:9040:8605:32b8")

        counters = self.stack.get_counters("sip")
        self.assertEqual(counters["requests"], 2)
        self.assertEqual(counters["responses"], 2)
        self.assertEqual(counters["registers"], 0)

        data_cache = self.stack.get_cache_data("SIP", "via")
        self.assertEqual(len(data_cache), 1)

    def test03(self):
        """Verify the integrity of the DNS traffic."""

        def callback_domain(_):
            self.called_callback += 1

        dom = pyaiengine.DomainName("Some domain", ".org")

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "dns")

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        self.assertEqual(self.called_callback, 1)

        data = json.loads(adaptor.lastdata)
        self.assertEqual(data["ip"]["src"], "2001:507:0:1:200:8600:0:1")
        self.assertEqual(data["ip"]["dst"], "2001:507:0:1:200:8600:0:2")
        self.assertEqual(data["dns"]["domain"], "itojun.org")

    def test04(self):
        """Verify the integrity of the SSL traffic."""

        def callback_domain(flow):
            self.called_callback += 1
            self.assertEqual(flow.ssl_info.server_name, "search.services.mozilla.com")
            # Is no issuer because the cert packet is just after
            self.assertEqual(flow.ssl_info.issuer_name, "")

        dom = pyaiengine.DomainName("Some domain", ".mozilla.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "SSL")

        adaptor = DatabaseTestAdaptor()

        self.stack.set_tcp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gtp_ip6_ssl.pcap")

        self.assertEqual(self.called_callback, 1)

        counters = self.stack.get_counters("SSL")
        self.assertEqual(counters["server hellos"], 1)
        self.assertEqual(counters["server dones"], 0)
        self.assertEqual(counters["records"], 11)
        self.assertEqual(counters["alerts"], 1)
        self.assertEqual(counters["client hellos"], 1)
        self.assertEqual(counters["handshakes"], 4)
        self.assertEqual(counters["certificates"], 1)

        data = json.loads(adaptor.lastdata)
        self.assertEqual(data["ssl"]["issuer"], "DigiCert SHA2 Secure Server CA")

    def test05(self):
        """Verify DomainName with IPSets on TCP traffic."""

        def callback_ipset(flow):
            self.called_callback += 1
            self.assertEqual(flow.ssl_info, None)

        def callback_domain(flow):
            self.called_callback += 1
            self.assertEqual(flow.ssl_info.server_name, "search.services.mozilla.com")

        dom = pyaiengine.DomainName("Some domain", ".mozilla.com")

        dman = pyaiengine.DomainNameManager()
        dom.callback = callback_domain
        dman.add_domain_name(dom)

        self.stack.set_domain_name_manager(dman, "ssl")

        ipset = pyaiengine.IPSet("IPv6 generic set", \
            ["2001:507:0:1:200:8600:0:2",
             "2001:507::1:200:8600:0:2",
             "2001:507:0:1:200:8600:0:100"])
        ipset.callback = callback_ipset
        im = pyaiengine.IPSetManager()

        im.add_ip_set(ipset)
        self.stack.tcp_ip_set_manager = im

        self.inject("../pcapfiles/gtp_ip6_ssl.pcap")

        self.assertEqual(self.called_callback, 2)

    def test06(self):
        """Verify Regex with IPSets on UDP traffic."""

        def callback_ipset(flow):
            self.called_callback += 1
            self.assertIsNotNone(flow)

        def callback_regex(flow):
            self.called_callback += 1
            self.assertIsNotNone(flow)

        rman = pyaiengine.RegexManager()

        reg = pyaiengine.Regex("Something", "^MESSAGE.*$", callback_regex)
        rman.add_regex(reg)
        self.stack.udp_regex_manager = rman

        self.stack.mode = "nids"

        ipset = pyaiengine.IPSet("IPv6 generic set", ["fd01::183", "2001:507:0:1:200:8600:0:100"])
        ipset.callback = callback_ipset
        iman = pyaiengine.IPSetManager()

        iman.add_ip_set(ipset)
        self.stack.udp_ip_set_manager = iman

        self.inject("../pcapfiles/gprs_ip6_udp.pcap")

        self.assertEqual(self.called_callback, 2)

    def test07(self):
        """Verify the change of execution by changing the regex_manager of the flow."""

        def callback1(flow):
            """
            On a regular execution, without changing the regex_manager, the flow
            will stop the check of new regex, because there is one that matches.
            However, by changing the matched RegexManager to other value we tell
            the engine to continue the exection but with the use of other RegexManager.
            """

            self.assertIsNotNone(flow.regex_manager)
            self.assertIsNotNone(flow.regex)
            self.assertEqual(flow.regex_manager.name, rman1.name)
            self.called_callback += 1
            flow.regex_manager = rman2

        def callback2(flow):
            self.assertIsNotNone(flow.regex_manager)
            self.assertEqual(flow.regex_manager.name, rman2.name)
            flow.regex_manager = rman3
            self.called_callback += 1

        def callback3(flow):
            self.assertIsNotNone(flow.regex_manager)
            self.assertEqual(flow.regex_manager.name, rman3.name)

            # Stop the execution
            flow.regex_manager = None
            self.called_callback += 1

        def callback4(flow):
            """ This callback is not called """
            self.called_callback += 1
            self.assertIsNotNone(flow)

        reg1 = pyaiengine.Regex("Rule1", "^SUBSCRIBE.*$", callback1)
        rother = pyaiengine.Regex("Rule1 extra", "^OTHER THING.*$")
        reg2 = pyaiengine.Regex("Rule2", "^(SIP/2.0 405).*$", callback2)
        reg3 = pyaiengine.Regex("Rule3", "^MESSAGE.*$", callback3)
        reg4 = pyaiengine.Regex("Rule4", "^(SIP/2.0 202).*$", callback4)

        rman1 = pyaiengine.RegexManager("one", [reg1, rother])
        rman2 = pyaiengine.RegexManager("two", [reg2])
        rman3 = pyaiengine.RegexManager("three", [reg3])

        adaptor = DatabaseTestAdaptor()
        self.stack.set_udp_database_adaptor(adaptor, 1)

        self.stack.udp_regex_manager = rman1
        self.stack.mode = "nids"

        self.inject("../pcapfiles/gprs_ip6_udp.pcap")

        self.assertEqual(reg1.matchs, 1)
        self.assertEqual(rother.matchs, 0)
        self.assertEqual(reg2.matchs, 1)
        self.assertEqual(reg3.matchs, 1)
        self.assertEqual(reg4.matchs, 0)
        self.assertEqual(self.called_callback, 3)

        data = json.loads(adaptor.all_data[1])
        # The first to records, that correspond to the first two packets,
        # should"t have any reference to the regex
        self.assertNotIn("matchs", data)
        data = json.loads(adaptor.all_data[2])
        self.assertNotIn("matchs", data)

        # The 3 and 4 packet should have the regex
        data = json.loads(adaptor.all_data[3])
        self.assertIn("matchs", data)
        self.assertEqual(data["matchs"], "Rule3")

        data = json.loads(adaptor.all_data[4])
        self.assertIn("matchs", data)
        self.assertEqual(data["matchs"], "Rule3")

    def test08(self):
        """Verify the functionality of the internal caches."""

        cache = self.stack.get_cache("dns", "domain")
        cache.destroy(cache.total_items)

        adaptor = DatabaseTestAdaptor()

        self.stack.set_udp_database_adaptor(adaptor)

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        data = json.loads(adaptor.lastdata)

        self.assertEqual(data["ip"]["src"], "2001:507:0:1:200:8600:0:1")
        self.assertEqual(data["ip"]["dst"], "2001:507:0:1:200:8600:0:2")

        self.assertNotIn("domain", data["dns"])

    def test09(self):
        """Verify classification with stack mode changes."""

        self.stack.mode = "frequency"
        
        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        counters = self.stack.get_counters("DNS")
        self.assertEqual(counters["packets"], 0)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 0)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 2)

        self.stack.udp_flow_manager.flush()

        self.stack.mode = "full"

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        counters = self.stack.get_counters("DNS")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 0)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 2)

        self.stack.udp_flow_manager.flush()

        self.stack.mode = "nids"

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        counters = self.stack.get_counters("DNS")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("UDPGENERIC")
        self.assertEqual(counters["packets"], 2)

        counters = self.stack.get_counters("UDPFREQUENCY")
        self.assertEqual(counters["packets"], 2)

    def test10(self):
        """Verify that the filtering works on the
        funtion show_flows with the next flows

        [5.233.130.56:1024]:17:[5.233.130.55:2152] gprs
        [2001:507::1:200:8600:0:1:2396]:17:[2001:507::1:200:8600:0:2:53] dns
        """

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="GPRS")
            file.seek(0)
            data = file.read()
            self.assertIn(b"[5.233.130.56:1024]:17:[5.233.130.55:2152]", data)
            self.assertNotIn(b"[2001:507::1:200:8600:0:1:2396]:17:[2001:507::1:200:8600:0:2:53]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(l7protocol="dns")
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[5.233.130.56:1024]:17:[5.233.130.55:2152]", data)
            self.assertIn(b"[2001:507:0:1:200:8600:0:1:2396]:17:[2001:507:0:1:200:8600:0:2:53]", data)

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_flows(dst_ip="2001:507:0:1:200:8600:0:2", dst_port=53)
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"[5.233.130.56:1024]:17:[5.233.130.55:2152]", data)
            self.assertIn(b"[2001:507:0:1:200:8600:0:1:2396]:17:[2001:507:0:1:200:8600:0:2:53]", data)

    def test11(self):
        """Verify the functionality when there is a cache fail."""

        self.called_callback = 0

        def fail_cache_callback(flow):
            self.called_callback += 1
            self.assertEqual("2001:507:0:1:200:8600:0:1:2396:17:2001:507:0:1:200:8600:0:2:53", str(flow))

        # Reduce the memory for store DNS information
        self.stack.decrease_allocated_memory("DNS", 2000)

        # Set the callback for be called on a cache fail
        self.stack.set_on_fail_cache_callback("DNS", fail_cache_callback)

        self.inject("../pcapfiles/gtp_ip6_dns.pcap")

        self.assertEqual(self.called_callback , 1)

    def test12(self):
        """Verify that we can disable protocols on the stack."""

        self.stack.disable_protocol("SIP")

        self.inject("../pcapfiles/gprs_ip6_udp.pcap")

        with tempfile.TemporaryFile() as file, stdout_redirected(file):
            self.stack.show_protocol_statistics()
            file.seek(0)
            data = file.read()
            self.assertNotIn(b"SIP", data)
            self.assertIn(b"UDPGeneric     2332", data)


class StackLanHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            "../pcapfiles/4udppackets.pcap")

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Test the /v1/protocols/summary URI."""

        uri = f"{self.baseuri}/v1/protocols/summary"
        headers = {}
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(res.text.startswith("Protocol statistics summary"), True)

        headers = {"Accept" : "application/json"}

        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Cache-Control", res.headers)
        self.assertEqual(res.headers["Cache-Control"], "no-store")

        keys = ["bytes", "cache_memory", "cpu_cycles", "events", "memory", "miss",
            "name", "packets", "used_memory"]

        for item in res.json():
            self.assertTrue(all(element in keys for element in item.keys()))

    def test02(self):
        """Test the /v1/flows URI."""

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/flows"
            headers = {}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.text.startswith("Flows on memory"), True)
            self.assertIn("Cache-Control", res.headers)
            self.assertEqual(res.headers["Cache-Control"], "no-store")

            params = {"limit": 1}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.text.startswith("Flows on memory"), True)

            params = {"limit": 1, "l7protocol": "http"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.text.startswith("Flows on memory"), True)

            params = {"limit": 100, "l7protocol": "dns"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.text.startswith("Flows on memory"), True)

    def test03(self):
        """Test the /v1/protocol/ URI with DNS."""

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/protocol"
            params = {"name": "dns"}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertTrue(res.text.startswith("DNS"))
            self.assertEqual(res.text.find("FlowForwarder"), -1)
            self.assertEqual(res.text.find("Name cache statistics"), -1)
            self.assertIn("Cache-Control", res.headers)
            self.assertEqual(res.headers["Cache-Control"], "no-store")

            params = {"name": "dns", "stats_level": 3}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertTrue(res.text.startswith("DNS"))
            self.assertEqual(res.text.find("Name cache statistics"), -1)

            params = {"name": "dns", "stats_level": 6}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertTrue(res.text.startswith("DNS"))
            self.assertGreater(res.text.find("FlowForwarder"), 0)
            self.assertGreater(res.text.find("Domain cache statistics"), 0)

            res = session.get(uri)
            self.assertEqual(res.status_code, 400)
            self.assertEqual(res.headers["Content-Type"], "text/html")
            self.assertIn("No protocol name set", res.text)

            headers = {"Accept": "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 400)

    def test04(self):
        """Test the /v1/protocol/ URI but with JSON format."""

        headers = {"Accept" : "application/json"}
        protocols = ["dns", "http", "ip", "tcp", "udp", "pop", "smtp", "bitcoin"]
        protocols += ["icmp", "imap", "ssl", "ssdp", "ssh", "modbus", "mqtt", "smb"]
        protocols += ["dcerpc", "tcpgeneric", "sip", "dhcp", "ntp", "snmp", "netbios"]
        protocols += ["coap", "rtp", "quic", "udpgeneric"]

        uri = f"{self.baseuri}/v1/protocol"
        with requests.Session() as session:
            for proto in protocols:
                params = {"name": proto, "stats_level": 5}
                res = session.get(uri, headers=headers, params=params)
                self.assertEqual(res.status_code, 200)
                jres = res.json()
                self.assertIn("name", jres)
                self.assertIn("dynamic_memory", jres)
                self.assertIn("allocated_bytes", jres)
                self.assertIn("packets", jres)
                self.assertIn("bytes", jres)
                self.assertIn("data_time", jres)
                self.assertIn("cpu_cycles", jres)

    def test05(self):
        """Test the /v1/summary URI."""

        headers = {}

        uri = f"{self.baseuri}/v1/summary"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Cache-Control", res.headers)
        self.assertEqual(res.headers["Cache-Control"], "no-store")

        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/summary"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertIn("stack", jres)
        self.assertIn("pcapfile", jres)
        self.assertEqual(jres["stack"], "Lan network stack")
        self.assertEqual(jres["shell"], False)

    def test06(self):
        """Test the /v1/system URI."""

        headers = {}

        uri = f"{self.baseuri}/v1/system"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Cache-Control", res.headers)
        self.assertEqual(res.headers["Cache-Control"], "max-age=350")
        self.assertTrue(res.text.startswith("System process statistics"))
        self.assertIn("Architecture:", res.text)
        self.assertIn("System:", res.text)
        self.assertIn("Pid:", res.text)
        self.assertIn("Voluntary ctx switches:", res.text)

        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/system"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        jres = res.json()

        # print(json.dumps(jres,sort_keys=True,indent=4, separators=(",", ": ")))

        self.assertIn("pid", jres)
        self.assertIn("euid", jres)
        self.assertIn("nodename", jres)
        self.assertIn("machine", jres)
        self.assertIn("release", jres)
        self.assertIn("elapsed_time", jres)
        self.assertIn("virtual_memory", jres)
        self.assertEqual(jres["euid"], os.geteuid())
        self.assertTrue(jres["elapsed_time"].startswith("00:00:00"))

        if (sys.platform == "linux"):
            self.assertIn("lock_memory_pages", jres)
            if os.geteuid() == 0:
                self.assertEqual(jres["lock_memory_pages"], True)
            else:
                self.assertEqual(jres["lock_memory_pages"], False)
            self.assertIn("resident_memory", jres)
            self.assertIn("shared_memory", jres)
            self.assertIn("unshared_data", jres)
            self.assertIn("unshared_stack", jres)
        else:
            self.assertNotIn("lock_memory", jres)
            self.assertNotIn("resident_memory", jres)
            self.assertNotIn("shared_memory", jres)
            self.assertNotIn("unshared_data", jres)
            self.assertNotIn("unshared_stack", jres)

    def test07(self):
        """ Test the /v1/uris URI """
        headers = {}

        uri = f"{self.baseuri}/v1/uris"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Cache-Control", res.headers)
        self.assertEqual(res.headers["Cache-Control"], "public")

        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/uris"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertIn("uris", jres)

    def test08(self):
        """ Test the /v1/protocol/http/cache/hosts URI """
        headers = {}

        file = open("../pcapfiles/accessgoogle.pcap", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        params = {"name": "http", "map": "hosts"}
        uri = f"{self.baseuri}/v1/protocol"
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)

        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)

        jres = res.json()

        self.assertIn("www.google.com", jres)

        params = {"name": "http", "map": "uris"}
        uri = f"{self.baseuri}/v1/protocol"
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)

        jres = res.json()

        self.assertIn("/textinputassistant/tia.png", jres)
        self.assertIn("/", jres)

    def test09(self):
        """ verify the process of upload pcap files """
        headers = {}

        file = open("./protocols/dns/packets/packet08.pcap", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # Get all flows in JSON format
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/flows"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

    def test10(self):
        """Verify the upload of python code."""
        headers = {"Content-Type" : "text/python"}

        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Python code loaded successfully", res.text)
        self.assertIn("<pre>6\n</pre>", res.text)

        # Force a syntax error
        code = "print(b)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 400)
        self.assertIn("Python code can not be loaded", res.text)
        self.assertIn("name 'b' is not defined", res.text)

    def test11(self):
        """Verify the URI /v1/flow/<flowid>"""

        with requests.Session() as session:
            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get all flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)

            flows = res.json()

            flow = None
            # Dont know if there will be more network flows on lo
            flow = [flow for flow in flows if flow["layer7"] == "DNS"]

            flow = flow[0]
            self.assertNotEqual(flow, None)

            flowid = "[%s:%d]" % (flow["ip"]["src"], flow["port"]["src"])
            flowid += ":%d:" % (flow["proto"])
            flowid += "[%s:%d]" % (flow["ip"]["dst"], flow["port"]["dst"])

            # Get the flow in plain format
            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Network flow", res.text)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:                    DNS", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("IP ttl:                    (128,0)", res.text)
            self.assertIn("Layer7: Domain:s2.youtube.com type:A tx:48938", res.text)

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            # Get the flow in JSON format but the flow dont exists
            headers = {"Accept" : "application/json"}

            flowid = "[%s:%d]" % (flow["ip"]["src"], flow["port"]["src"])
            flowid += ":%d:" % (flow["proto"])
            flowid += "[%s:%d]" % (flow["ip"]["dst"], flow["port"]["dst"] + 1)

            headers = {"Accept" : "text/plain"}
            params = {"id": flowid}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 404)
            self.assertIn("Network flow not set or found", res.text)

            params = {"id": ""}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 404)
            self.assertEqual(res.headers["Content-Type"], "text/html")
            self.assertIn("Network flow not set or found", res.text)

    def test12(self):
        """Verify the URI PUT /v1/flow/<flowid> with parameter for update the flow """
        flowid = "[192.168.1.1:63139]:17:[192.168.1.254:53]"

        headers = {}

        with requests.Session() as session:
            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertNotIn("label", jres)
            self.assertIn("reject", jres)
            self.assertEqual(jres["reject"], False)

            # Do a put and modify the value of the label and also
            # reject the flowl
            data = {"label" : "This is a lovely label my friend",
                    "reject" : True}
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertIn("label", jres)
            self.assertEqual(jres["label"], data["label"])
            self.assertIn("reject", jres)
            self.assertEqual(jres["reject"], True)

            # Now with urlencode format on the body
            data = {
                "label" : "This is another lovely label",
                "reject" : False,
                "evidence": True,
                "accept": False
            }
            res = session.put(uri, data=data, params=params)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["label"], data["label"])
            self.assertEqual(flow["reject"], data["reject"])
            self.assertEqual(flow["evidence"], data["evidence"])
            self.assertEqual(flow["accept"], data["accept"])


    def test13(self):
        """Test the /v1/locals URI"""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python"}
            variable_name = "some_variable"
            code = "%s = 1 + 5" % variable_name
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            headers = {}
            uri = f"{self.baseuri}/v1/locals"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Python objects", res.text)

            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/locals"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()

            self.assertIn("objects", jres)

    def test14(self):
        """
        Full test that interacts with different uris to modify
        the behavior of the running engine.
    
        First we create a new regular expression for detection
        that will have a callback also and label the flow.
        """

        code = "def regex_callback(flow):\n"
        code += "    flow.label = 'IRC pass detected'\n"

        headers = {"Content-Type" : "text/python"}
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 200)

        # The callback definition should be on memory
        del headers["Content-Type"]
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/locals"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        jres = res.json()

        # This has been tested manually
        # self.assertIn("regex_callback", jres["objects"])

    def test15(self):
        """Verify some of the SSL caches are accesible"""

        with requests.Session() as session:
            file = open("../pcapfiles/alibaba.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get all flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            req = session.get(uri, headers=headers)
            self.assertEqual(req.status_code, 200)

            jres = req.json()

            self.assertEqual(len(jres), 3) # There is the extra udp flow of the __init__
            self.assertEqual(jres[0]["ip"]["src"], "10.42.0.211")
            self.assertEqual(jres[1]["ip"]["src"], "10.42.0.211")
            self.assertTrue(jres[0]["upstream"]["bytes"] > 1024)
            self.assertEqual(jres[0]["upstream"]["packets"], 6)
            self.assertTrue(jres[1]["downstream"]["bytes"] > 0)
            self.assertEqual(jres[1]["downstream"]["packets"], 3)

            if defined("HAVE_JA3"):
                self.assertEqual(jres[0]["ssl"]["fingerprint"], "2b356f3492005681e356a72c1e4fd879")
                self.assertEqual(jres[1]["ssl"]["fingerprint"], "2b356f3492005681e356a72c1e4fd879")

            self.assertEqual(jres[0]["ssl"]["session"], 
                             "d7589284b1eecb5896a463486bace2a6569170c2e6bfa5e8329f93ef7806704c")
            self.assertEqual(jres[1]["ssl"]["session"], 
                             "d7589284b1eecb5896a463486bace2a6569170c2e6bfa5e8329f93ef7806704c")
            self.assertEqual(jres[0]["ssl"]["host"], "gw.alicdn.com")
            self.assertEqual(jres[1]["ssl"]["host"], "gw.alicdn.com")

            uri = f"{self.baseuri}/v1/protocol"
            params = {"name": "ssl", "map": "hosts"}
            res = requests.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertIn("gw.alicdn.com", jres)

            params = {"name": "ssl", "map": "sessions"}
            res = requests.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertIn("d7589284b1eecb5896a463486bace2a6569170c2e6bfa5e8329f93ef7806704c", jres)

            if defined("HAVE_JA3"):
                params = {"name": "ssl", "map": "fingerprints"}
                res = requests.get(uri, headers=headers, params=params)
                self.assertEqual(res.status_code, 200)
                jres = res.json()
                self.assertIn("2b356f3492005681e356a72c1e4fd879", jres)

    def test16(self):
        """Verify the functionality for show the current packet."""

        headers = {}

        file = open("./protocols/dns/packets/packet08.pcap", "rb")
        files = {"file" : file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # Get the current packet flow in text format
        uri = f"{self.baseuri}/v1/current_packet"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        self.assertIn("Flow: 192.168.1.1:63139:17:192.168.1.254:53", res.text)

        # Same but in JSON format

        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertIn("flow", jres)
        self.assertEqual(jres["flow"], "192.168.1.1:63139:17:192.168.1.254:53")
        self.assertEqual(jres["packet"][14], 69)

    def test17(self):
        """Verify the detach functionality of the Flow."""

        flowid = "[192.168.1.1:63139]:17:[192.168.1.254:53]"
        params = {"id": flowid}

        with requests.Session() as session:
            headers = {}

            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertEqual(jres["dns"]["domain"], "s2.youtube.com")
            self.assertEqual(jres["layer7"], "DNS")

            # Do a put and detach flow 
            # reject the flow
            data = {"detach" : True}
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertNotIn("dns", jres)
            self.assertEqual(jres["layer7"], "None")

    def test18(self):
        """Verify the parameters on the uri for retrieve
        network flows to the user."""

        file = open("../pcapfiles/alibaba.pcap", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # Get all TCP flows in JSON format
        params = {"protocol": 6}
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/flows"
        req = requests.get(uri, headers=headers, params=params)
        self.assertEqual(req.status_code, 200)

        jres = req.json()
        self.assertEqual(len(jres), 2)
        for flow in jres:
            self.assertEqual(flow["proto"], 6)

        params = {"protocol": 6, "dstip": "64.71.142.82"}
        req = requests.get(uri, headers=headers, params=params)
        self.assertEqual(req.status_code, 200)
        jres = req.json()

        self.assertEqual(len(jres), 1)
        for flow in req.json():
            self.assertEqual(flow["ip"]["dst"], "64.71.142.82")

    def test19(self):
        """Verify the parameters on the uri for retrieve
        network flows to the user with multiple pcaps."""

        pcapfiles = ["../pcapfiles/accessgoogle.pcap",
                     "../pcapfiles/4udppackets.pcap" ]
        uri = f"{self.baseuri}/v1/pcapfile"

        with requests.Session() as session:
            for entry in pcapfiles:
                with open(entry, "rb") as file:
                    files = {"file": file}
                    res = session.post(uri, files=files)
                    self.assertEqual(res.status_code, 200)

            # Get all TCP flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"protocol": 6}
            uri = f"{self.baseuri}/v1/flows"
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)

            jres = req.json()
            self.assertEqual(len(jres), 1)
            for flow in jres:
                self.assertEqual(flow["proto"], 6)

            params = {"protocol": 17}
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            jres = req.json()

            self.assertEqual(len(jres), 2)

            # The protocol 15 dont exists , just supported 6 and 17 so the system
            # will show all the flows
            params = {"protocol": 15}
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            jres = req.json()

            self.assertEqual(len(jres), 3)

            params = {"protocol": 17}
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            jres = req.json()

            self.assertEqual(len(jres), 2)

            params = {"protocol": 17, "l7protocol": "dns"}
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            jres = req.json()

            self.assertEqual(len(jres), 1)
            for flow in jres:
                self.assertEqual(flow["layer7"], "DNS")

    def test20(self):
        """Verify that we can see the properties of an object."""

        headers = {}
        uri = f"{self.baseuri}/v1/locals"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Python objects", res.text)

        params = {"name": "pepe"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 404)

        params = {"name": "stack"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        self.assertIn("name = Lan network stack", res.text)

    def test21(self):
        """Verify that we can see the properties of an object in json."""

        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/locals"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertIn("objects", data)
        self.assertIn("pcapfile", data["objects"])
        self.assertEqual("str", data["objects"]["pcapfile"])

        params = {"name": "stack"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertIn("properties", data)
        self.assertIn("name", data["properties"])
        self.assertEqual(data["properties"]["name"], "Lan network stack")
        self.assertEqual(data["properties"]["mode"], "full")

    def test22(self):
        """Verify that we can see one property of an object in json."""

        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/locals"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertIn("objects", data)
        self.assertIn("pcapfile", data["objects"])
        self.assertEqual("str", data["objects"]["pcapfile"])

        params = {"name": "stack", "property": "name"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertIn("properties", data)
        self.assertIn("name", data["properties"])
        self.assertEqual(data["properties"]["name"], "Lan network stack")
        self.assertEqual(len(data["properties"]), 1)

    def test23(self):
        """Verify that we can modify a property of an object."""

        with requests.Session() as session:
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/locals"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertIn("objects", data)
            self.assertIn("pcapfile", data["objects"])
            self.assertEqual("str", data["objects"]["pcapfile"])

            params = {"name": "stack", "property": "mode"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertIn("properties", data)
            self.assertIn("mode", data["properties"])
            self.assertEqual(data["properties"]["mode"], "full")
            self.assertEqual(len(data["properties"]), 1)

            params = {"name": "stack", "property": "mode", "value": "nids"}
            # Now change the value of the property
            res = session.put(uri, headers=headers, params=params)
            # WARNING check this on a integration test
            # self.assertEqual(res.status_code, 200)

            # Verify that the property has been changed
            params = {"name": "stack", "property": "mode"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertIn("properties", data)
            self.assertIn("mode", data["properties"])

            # WARNING Test this manually
            # self.assertEqual(data["properties"]["mode"], "nids")
            self.assertEqual(len(data["properties"]), 1)

    def test24(self):
        """Verify the access to the logs via HTTP."""

        uri = f"{self.baseuri}/v1/locals"
        params = {"name": "stack", "property": "mode"}
        with requests.Session() as session:
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)

            # Now access to the logs
            uri = f"{self.baseuri}/v1/logs"
            res = session.get(uri)
            lines = res.text.split("\n")
            total_lines = len(lines)
            self.assertIn("Processing packets from pcap file", lines[0])
            self.assertIn("GET /v1/locals?name=stack", lines[2])

            # Now access to the logs with limit
            params = {"limit": 1}
            uri = f"{self.baseuri}/v1/logs"
            res = session.get(uri, params=params)
            lines = res.text.split("\n")
            self.assertEqual(len(lines), 2)
            self.assertIn("GET /v1/logs ", lines[0])

    def test25(self):
        """Verify the access to the URI health."""

        uri = f"{self.baseuri}/v1/health"
        res = requests.get(uri)
        self.assertEqual(res.status_code, 200)
        self.assertEqual("Up and running ;)", res.text)

        version = ""
        with open("../config.h", "r") as fd_ver:
            for line in fd_ver.readlines():
                if (line.startswith("#define VERSION")):
                    version = "AIEngine " + line.split(" ")[2].replace("\"", "").rstrip()

        self.assertTrue(version != "")
        self.assertEqual(res.headers["Server"], version)

    def test26(self):
        """Verify that we can access to the anomalies generated
        on the network stack."""

        uri = f"{self.baseuri}/v1/anomalies"
        res = requests.get(uri)
        self.assertEqual(res.status_code, 200)
        data = res.text
        anomalies = {}
        lines = res.text.split("\n")
        for line in lines[1:-1]:
            item = line.replace("Total", "").lstrip().split(":")
            key = item[0]
            hits = item[1].strip()
            anomalies[key] = int(hits)

        self.assertIn("Anomalies", data)
        self.assertIn("IPv4 Fragmentation", data)
        self.assertIn("IPv6 Loop ext headers", data)
        self.assertIn("CoAP bogus headers", data)

        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertIn("anomalies", data)
        self.assertEqual(len(data["anomalies"]), 21)
        for item in data["anomalies"]:
            self.assertIn(item["name"], anomalies)
            self.assertEqual(anomalies[item["name"]], item["hits"])

    def test27(self):
        """Verify that we can access to the uris even if we
        dont put the host field on the requests correctly."""

        headers = {"Host": ""}
        uri = f"{self.baseuri}"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn(uri, res.text)

    def test28(self):
        """Verify that we can show the python help via API."""

        uri = f"{self.baseuri}/v1/help"
        res = requests.get(uri)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Help on module pyaiengine:", res.text)

    def test29(self):
        """Test the output of /v1/protocol/ URI"""

        data = {"dns":  "Total type others:",
                "http": "Total others:",
                "ip": "Total fragment packets:",
                "tcp": "Total rsts:",
                "udp": "Total invalid packets:",
                "pop": "Total quits:",
                "smtp": "Total starttls:",
                "bitcoin": "Total get blocks:",
                "icmp": "Total address replays:",
                "imap": "Total logouts:",
                "ssl": "Total handshakes finish:",
                "ssdp": "Total others:",
                "ssh": "Total encrypted bytes:",
                "modbus": "Total write multi regs:",
                "mqtt": "Total reserveds:",
                "smb": "Total nt creates:",
                "dcerpc": "Total orphaneds:",
                "tcpgeneric": "Total valid packets:",
                "sip": "Total infos:",
                "dhcp": "Total informs:",
                "ntp": "Total broadcasts:",
                "snmp": "Total set requests:",
                "netbios": "Total invalid packets:",
                "coap": "Total delete:",
                "rtp": "Total packets:",
                "quic": "Total valid packets:",
                "udpgeneric": "Total invalid packets:"}

        uri = f"{self.baseuri}/v1/protocol"
        with requests.Session() as session:
            for proto, value in data.items():
                params = {"name": proto, "stats_level": 4}
                res = session.get(uri, params=params)
                self.assertEqual(res.status_code, 200)
                self.assertIn(value, res.text)
                self.assertIn("Data time", res.text)

    def test30(self):
        """Test the /v1/protocol/ URI with max level."""

        protocols = ["dns", "http", "tcp", "udp", "pop", "smtp", "bitcoin"]
        protocols += ["imap", "ssl", "ssdp", "ssh", "modbus", "mqtt", "smb"]
        protocols += ["dcerpc", "tcpgeneric", "sip", "dhcp", "ntp", "snmp", "netbios"]
        protocols += ["coap", "rtp", "quic", "udpgeneric"]

        uri = f"{self.baseuri}/v1/protocol"
        with requests.Session() as session:
            for proto in protocols:
                params = {"name": proto, "stats_level": 10}
                res = session.get(uri, params=params)
                self.assertEqual(res.status_code, 200)
                self.assertIn("FlowForwarder", res.text)

    def test31(self):
        """Verify POST and PUT operations on non existing uris."""

        params = {"stats_level": 10}
        uri = f"{self.baseuri}/v1/bubu"
        res = requests.post(uri, params=params)
        self.assertEqual(res.status_code, 400)
        self.assertEqual("Invalid POST resource '/v1/bubu?stats_level=10'", res.text)

        res = requests.put(uri, params=params)
        self.assertEqual(res.status_code, 400)
        self.assertEqual("Invalid PUT resource '/v1/bubu?stats_level=10'", res.text)

    def test32(self):
        """Verify that the delay on the generated script
        changes depending if the execution is correct or wrong."""

        headers = {"Content-Type" : "text/python"}
        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Python code loaded successfully", res.text)
        self.assertIn("<pre>6\n</pre>", res.text)
        self.assertIn("<script>", res.text)
        self.assertIn("setTimeout(function()", res.text)
        self.assertIn("5000);", res.text)
        self.assertIn("</script>", res.text)

        code = "print(bubu)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 400)
        self.assertIn("Python code can not be loaded", res.text)
        self.assertIn("<script>", res.text)
        self.assertIn("setTimeout(function()", res.text)
        self.assertIn("3000);", res.text)

    def test33(self):
        """Verify that crap python code dont break the engine."""

        headers = {"Content-Type" : "text/python"}
        code = "print(bubu babaa)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 400)
        self.assertIn("Python code can not be loaded", res.text)
        self.assertIn("invalid syntax", res.text)
        self.assertIn("<script>", res.text)
        self.assertIn("setTimeout(function()", res.text)
        self.assertIn("3000);", res.text)

        code = "asdf;asdkfier print(bubu babaa)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 400)

    def test34(self):
        """Verify that the duration of the flow is correct."""

        file = open("../pcapfiles/two_http_flows_noending.pcap", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # The pcap file contains two flows
        # [192.168.1.13:55354]:6:[95.100.96.48:80] duration of 28 seconds
        # [192.168.1.13:49503]:6:[95.100.96.10:80] duration of 2 seconds
        with requests.Session() as session:
            flowid = "[192.168.1.13:55354]:6:[95.100.96.48:80]"
            params = {"id": flowid}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      TCP", res.text)
            self.assertIn("L7Protocol:                   HTTP", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("IP ttl:                    (64,56)", res.text)
            self.assertIn("Duration:              00:00:00:28", res.text)
            self.assertIn("TCP state:             ESTABLISHED", res.text)
            self.assertIn("Layer7: Req(39)Res(38)Code(200) Host:www.wired.com", res.text)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("duration", jdata)
            self.assertEqual(jdata["duration"], 28)

            flowid = "[192.168.1.13:49503]:6:[95.100.96.10:80]"
            params = {"id": flowid}
            headers = {"Accept" : "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Duration:              00:00:00:02", res.text)
            self.assertIn("TCP state:             ESTABLISHED", res.text)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("duration", jdata)
            self.assertEqual(jdata["duration"], 2)

    def test35(self):
        """Verify that access to the evidences url dont work
        if is not enable on the PacketDispatcher."""

        uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
        res = requests.get(uri)
        self.assertEqual(res.status_code, 404)
        self.assertIn("Evidences not available", res.text)
        self.assertIn("<script>", res.text)
        self.assertIn("setTimeout(function()", res.text)
        self.assertIn("3000);", res.text)

    def test36(self):
        """Verify that we can show the python help of
        a specific class via API."""

        params = {"class": "Flow"}
        uri = f"{self.baseuri}/v1/help"
        res = requests.get(uri, params=params)
        self.assertEqual(res.status_code, 200)
        self.assertTrue(res.text.startswith("Help on class Flow in module pyaiengine:"))

    def test37(self):
        """Verify create a new Domain that matchs, generates
        extra information on the DNS"""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python"}
            code = "dom = pyaiengine.DomainName('Google', '.google.com')\n"
            code += "dm = pyaiengine.DomainNameManager([dom])\n"
            code += "stack.set_domain_name_manager(dm, 'dns')"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Python code loaded successfully", res.text)

            file = open("../pcapfiles/accessgoogle.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            flowid = "[192.168.1.13:54737]:17:[89.101.160.5:53]"
            params = {"id": flowid}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("74.125.24.99", res.text)
            self.assertIn("74.125.24.105", res.text)
            self.assertIn("74.125.24.147", res.text)
            self.assertIn("74.125.24.106", res.text)
            self.assertIn("74.125.24.104", res.text)
            self.assertIn("74.125.24.103", res.text)
            self.assertIn("2a00:1450:400b:c02::69", res.text)

    def test38(self):
        """Verify the upload of python code with a delay setup on the URI."""
        headers = {"Content-Type" : "text/python"}

        code = "a = 1 + 5" + "\n" + "print(a)"
        params = {"delay": 10000}
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        self.assertIn("Python code loaded successfully", res.text)
        self.assertIn("<pre>6\n</pre>", res.text)
        self.assertIn("}, 10000);", res.text)

        # Force a syntax error
        code = "print(b)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 400)
        self.assertIn("Python code can not be loaded", res.text)
        self.assertIn("name 'b' is not defined", res.text)
        self.assertIn("}, 3000);", res.text)

    def test39(self):
        """Verify that we can enable vlan on the stack and access
        and modify the existing flows with vlan headers."""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python"}

            code = "stack.link_layer_tag = 'vlan'"
            params = {"delay": 10000}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            file = open("../pcapfiles/flow_vlan_x11.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            flowid = "[131.151.32.21:6000]:6:[131.151.32.129:1173]"
            params = {"id": flowid, "tag": 32}
            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertNotIn("label", jres)
            self.assertEqual(jres["evidence"], False)
            self.assertEqual(jres["layer7"], "TCPGeneric")

            # Modify the label of the flow
            data = {"label" : "im a lovely label"}
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn(data["label"], res.text)

    def test40(self):
        """Verify that the Data time of a protocol change
        by using the API."""

        with requests.Session() as session:
            file = open("../pcapfiles/accessgoogle.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            params = {"name": "http"}
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("data_time", jdata)
            boot_time = API_DELAY - 1
            self.assertEqual(jdata["data_time"], f"00:00:00:0{boot_time}")
            self.assertGreater(jdata["cpu_cycles"], 0)

            elapsed = random.randint(1, 3)
            time.sleep(elapsed)

            params = {"name": "http"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("data_time", jdata)
            self.assertEqual(jdata["data_time"], f"00:00:00:0{elapsed + boot_time}")

            headers = {"Content-Type" : "text/python"}
            code = "stack.release_cache('http')"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            params = {"name": "http"}
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("data_time", jdata)
            self.assertEqual(jdata["data_time"], "00:00:00:00")

            params = {"name": "ip"}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("data_time", jdata)
            self.assertEqual(jdata["data_time"], f"00:00:00:0{elapsed + boot_time}")

    def test41(self):
        """Verify that the Data time of a FlowManager change
        by using the API."""

        boot_time = API_DELAY - 1
        with requests.Session() as session:
            file = open("../pcapfiles/accessgoogle.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get all TCP flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"protocol": 6}
            uri = f"{self.baseuri}/v1/flows"
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            self.assertEqual(len(req.json()), 1)

            # Shows the TCP FlowManager
            headers = {"Content-Type" : "text/python"}
            code = "stack.tcp_flow_manager.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn(f"Data time:             00:00:00:0{boot_time}", res.text)

            elapsed = random.randint(1, 3)
            time.sleep(elapsed)

            headers = {"Content-Type" : "text/python"}
            code = "stack.tcp_flow_manager.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn(f"Data time:             00:00:00:0{elapsed + boot_time}", res.text)

            headers = {"Content-Type" : "text/python"}
            code = "stack.tcp_flow_manager.flush()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            headers = {"Content-Type" : "text/python"}
            code = "stack.tcp_flow_manager.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Data time:             00:00:00:00", res.text)

            # Get all TCP flows in JSON format again
            headers = {"Accept" : "application/json"}
            params = {"protocol": 6}
            uri = f"{self.baseuri}/v1/flows"
            req = session.get(uri, headers=headers, params=params)
            self.assertEqual(req.status_code, 200)
            self.assertEqual(req.json(), None)

    def test42(self):
        """Verify that the url protocol control the amount of 
        data that can be retrieved by using the limit parameter."""

        with requests.Session() as session:
            file = open("../pcapfiles/accessgoogle.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            uri = f"{self.baseuri}/v1/protocol"
            params = {"name": "http", "stats_level": 5, "limit": 10}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Uri:/textinputassistant/tia.png:1", res.text)
            self.assertIn("Host:www.google.com:1", res.text)
            self.assertIn("ContentType:text/html:2", res.text)

            # Now data should be filtered
            params = {"name": "http", "stats_level": 5, "limit": 0}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertNotIn("Uri:/textinputassistant/tia.png:1", res.text)
            self.assertNotIn("Host:www.google.com:1", res.text)
            self.assertNotIn("ContentType:text/html:2", res.text)
            self.assertIn("...", res.text)

            headers = {"Accept" : "application/json"}
            params = {"name": "http", "stats_level": 5, "limit": 1, "map": "uris"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertIn("/", data)
            self.assertEqual(data["/"], 1)

    def test43(self):
        """Verify that we can send alarm data to the engine."""

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/alarms"
            data = "hola"
            res = session.post(uri, data=data)
            self.assertEqual(res.status_code, 415)

            data = dict()
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 404)

            # Now we create a callback on the fly
            # The callback will create a file on the filesystem
            # with the json received

            filename = "jsoncontent.json"
            if os.path.exists(filename):
                os.remove(filename)

            headers = {"Content-Type" : "text/python"}
            code = "def alarm_callback(data):\n"
            code += f"    with open('{filename}', 'w') as fd:\n"
            code += "        fd.write(data)\n"
            code += "pdis.alarm_callback = alarm_callback"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            uri = f"{self.baseuri}/v1/alarms"
            data = {"some": "data", "items": [1, 2, 3]}
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 200)

            # Now we verify that the file exists and the
            # content is json
            self.assertTrue(os.path.exists(filename))

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test44(self):
        """Verify that we can receive an alarm that the engine generates."""

        label = "".join(random.choices(string.ascii_uppercase + string.digits, k=32))

        with requests.Session() as session:
            # First we generate a basic dns callback
            headers = {"Content-Type" : "text/python"}
            code = "def callback_dns(flow):\n"
            code += "    flow.alerted = True\n"
            code += f"    flow.label = '{label}'\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom = pyaiengine.DomainName(\"Domain1\", \".youtube.com\")\n"
            code += "dom.callback = callback_dns\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Now we create a callback on the fly
            # The callback will create a file on the filesystem
            # with the json received

            filename = "jsoncontent.json"
            if os.path.exists(filename):
                os.remove(filename)

            code = "def alarm_callback(data):\n"
            code += "    print(data)\n"
            code += f"    with open(\"{filename}\", \"w\") as fd:\n"
            code += "        fd.write(data)\n"
            code += "pdis.alarm_callback = alarm_callback"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Now we inject the traffic
            file = open("./protocols/dns/packets/packet09.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Now we sleep and reads from the queue to see message
            # generated by the alerted property and we send back
            # to the engine via alert uri
            time.sleep(QUEUE_DELAY)
            cmd = ["./alarmreader", "-H", f"{HTTP_LISTEN_IP}", "-t", "1", "-p", f"{self.port}", "-u", "/v1/alarms"]
            ret = subprocess.run(cmd)

            self.assertTrue(os.path.exists(filename))
            with open(filename, "r") as fd:
                data = json.loads(fd.read())
            self.assertEqual(data["dns"]["domain"], "s2.youtube.com")
            self.assertEqual(data["label"], label)

    def test45(self):
        """Verify that the user can see the L7 payloads of the
        flow if that has been requested over the Rest API."""

        with requests.Session() as session, \
            open("../pcapfiles/4udppackets.pcap", "rb") as file:

            # The user sets the payload for see the packets of the flow
            flowid = "[10.0.2.15:51413]:17:[88.190.242.141:6881]"
            params = {"id": flowid, "payload": True}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertNotIn("d1:rd2:id20:H.U", res.text)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid, "payload": True, "len": 1500}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("d1:rd2:id20:H.U", res.text)
            self.assertIn("5:nodes208", res.text)
            self.assertIn("y1:r", res.text)

            params = {"id": flowid, "payload": True}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("d1:rd2:id20:H.U", res.text)
            self.assertIn("5:nodes208", res.text)
            self.assertIn("y1:r", res.text)

            params = {"id": flowid, "payload": True, "len": 64}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("d1:rd2:id20:H.U", res.text)
            self.assertIn("5:nodes208", res.text)
            self.assertNotIn("y1:r", res.text)

            params = {"id": flowid, "payload": True, "len": 16}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("d1:rd2:id20:H.U", res.text)
            self.assertNotIn("5:nodes208", res.text)
            self.assertNotIn("y1:r:", res.text)

            params = {"id": flowid, "payload": True, "len": 16}
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertIn("l7_payload", flow)
            self.assertEqual(len(flow["l7_payload"]), 16)

            cad = "".join(str(chr(x)) for x in flow["l7_payload"])
            self.assertTrue(cad.startswith("d1:rd2:id20:HÍU"))

    def test46(self):
        """Verify that if the alarm callback throws an exception
        the alarm callback is disabled properly."""

        with requests.Session() as session:
            # Now we create a callback that will have an exception

            filename = "jsoncontent.json"
            if os.path.exists(filename):
                os.remove(filename)

            headers = {"Content-Type" : "text/python"}
            code = "def alarm_callback(data):\n"
            code += f"    import json\n"
            code += f"    jdata = json.loads(data)\n"
            code += f"    _ = jdata[\"dont_exist\"]\n"
            code += f"    with open('{filename}', 'w') as fd:\n"
            code += "        fd.write(data)\n"
            code += "pdis.alarm_callback = alarm_callback"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # This request will generate an exception
            uri = f"{self.baseuri}/v1/alarms"
            data = {"some": "data", "items": [1, 2, 3]}
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 400)
            self.assertIn("Alarm callback has been disable", res.text)

            # Now we verify that the file dont exists
            self.assertFalse(os.path.exists(filename))

            # There is no callback for the alarms
            uri = f"{self.baseuri}/v1/alarms"
            data = {"some": "data", "items": [1, 2, 3]}
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 404)

            # Now we reload the code again
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # We send data that wont generate exception
            uri = f"{self.baseuri}/v1/alarms"
            data = {"some": "data", "items": [1, 2, 3], "dont_exist": True}
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 200)

            self.assertTrue(os.path.exists(filename))

    def test47(self):
        """Verify that we can upoad data by using the alarm callback
        and add some domains to a DomainNameManager that has been
        created and pluged over the Rest API."""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python"}
            code = "dm = pyaiengine.DomainNameManager()\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            code = "dm.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DomainNameManager (Generic Domain Name Manager) Plugged on DNS", res.text)

            headers = {"Content-Type" : "text/python"}
            code = "def alarm_callback(data):\n"
            code += f"    import json\n"
            code += f"    jdata = json.loads(data)\n"
            code += "    for name in jdata[\"names\"]:\n"
            code += "        dom = pyaiengine.DomainName(\"Domain1\", name)\n"
            code += "        dm.add_domain_name(dom)\n"
            code += "pdis.alarm_callback = alarm_callback"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            uri = f"{self.baseuri}/v1/alarms"
            data = {"names": [".youtube.com", "gmail.com"]}
            res = session.post(uri, json=data)
            self.assertEqual(res.status_code, 200)

            code = "dm.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DomainNameManager (Generic Domain Name Manager) Plugged on DNS", res.text)
            self.assertIn(data["names"][0], res.text)
            self.assertIn(data["names"][1], res.text)
            self.assertNotIn("Matchs:1", res.text)

            # Inject a DNS query
            file = open("./protocols/dns/packets/packet09.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            code = "dm.show()"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DomainNameManager (Generic Domain Name Manager) Plugged on DNS", res.text)
            self.assertIn(data["names"][0], res.text)
            self.assertIn(data["names"][1], res.text)
            self.assertIn("Matchs:1", res.text)

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test48(self):
        """Verify that we can receive an alarm that has been trunkated."""

        label = "".join(random.choices(string.ascii_uppercase + string.digits, k=1024))

        with requests.Session() as session:
            # First we generate a basic dns callback
            headers = {"Content-Type" : "text/python"}
            code = "def callback_dns(flow):\n"
            code += "    flow.alerted = True\n"
            code += f"    flow.label = '{label}'\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom = pyaiengine.DomainName(\"Domain1\", \".youtube.com\")\n"
            code += "dom.callback = callback_dns\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Now we create a callback on the fly
            # The callback will create a file on the filesystem
            # with the json received

            filename = "jsoncontent.json"
            if os.path.exists(filename):
                os.remove(filename)

            code = "def alarm_callback(data):\n"
            code += f"    with open(\"{filename}\", \"w\") as fd:\n"
            code += "        fd.write(data)\n"
            code += "pdis.alarm_callback = alarm_callback"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Now we inject the traffic
            file = open("./protocols/dns/packets/packet09.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Now we sleep and reads from the queue to see message
            # generated by the alerted property and we send back
            # to the engine via alert uri
            time.sleep(QUEUE_DELAY)
            cmd = ["./alarmreader", "-H", f"{HTTP_LISTEN_IP}", "-t", "1", "-p", f"{self.port}", "-u", "/v1/alarms"]
            ret = subprocess.run(cmd)

            self.assertTrue(os.path.exists(filename))
            with open(filename, "r") as fd:
                data = json.loads(fd.read())
            self.assertEqual(data["label"], "Data has been removed")


class StackOpenFlowHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interfacei
    with a network stack for OpenFlow."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackOpenFlow", cls.port,
            "../pcapfiles/openflow_dns.pcap")

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify that can see all the flows."""

        keys = ["[192.168.1.2:57328]:6:[192.168.1.1:6633]",
                "[129.21.3.17:53]:17:[192.168.2.6:28848]",
                "[192.168.2.6:8462]:17:[129.21.3.17:53]"]

        uri = f"{self.baseuri}/v1/flows"
        res = requests.get(uri)
        self.assertEqual(res.status_code, 200)
        for value in keys:
            self.assertIn(value, res.text)

        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(len(res.json()), 3)

    def test02(self):
        """Verify that can see the openflow flow
        and modify a property."""

        with requests.Session() as session:
            flowid = "[192.168.1.2:57328]:6:[192.168.1.1:6633]"
            params = {"id": flowid}
            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertNotIn("label", jres)
            self.assertEqual(jres["evidence"], False)
            self.assertEqual(jres["layer7"], "OpenFlow")

            # Modify the label of the flow
            data = {"label" : "im a lovely label"}
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(jres["label"], "im a lovely label")

            headers = {"Accept" : "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Label:           im a lovely labe", res.text)

    def test03(self):
        """Verify that can see the high flow
        and modify a property, and reset."""

        flowid = "[192.168.2.6:8462]:17:[129.21.3.17:53]"

        with requests.Session() as session:
            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertNotIn("label", jres)
            self.assertEqual(jres["evidence"], False)
            self.assertEqual(jres["layer7"], "DNS")
            self.assertIn("dns", jres)
            self.assertIn("domain", jres["dns"])
            self.assertEqual(jres["dns"]["domain"], "daisy.ubuntu.com")

            headers = {"Content-Type" : "text/python"}
            code = "stack.release_cache(\"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertNotIn("label", jres)
            self.assertEqual(jres["evidence"], False)
            self.assertEqual(jres["layer7"], "DNS")
            self.assertNotIn("dns", jres)

    def test04(self):
        """Generate new DomainNameManager to see
        how traffic will match with some DomainName."""

        with requests.Session() as session:
            # First reset all the flows
            headers = {"Content-Type" : "text/python"}
            code = "stack.udp_flow_manager.flush()\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Create the intelligence rules
            code = "dm = pyaiengine.DomainNameManager()\n"
            code += "dom = pyaiengine.DomainName(\"Domain1\", \".ubuntu.com\")\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # Check that all is created on the engine
            uri = f"{self.baseuri}/v1/locals"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Python objects", res.text)
            self.assertIn("dm: DomainNameManager", res.text)
            self.assertIn("dom: DomainName", res.text)

            # Now inject the pcap file
            file = open("../pcapfiles/openflow_dns.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Check the output
            headers = {"Content-Type" : "text/python"}
            code = "dm.show()\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DomainNameManager (Generic Domain Name Manager) Plugged on DNS", res.text)
            self.assertIn("Matchs:1", res.text)

    def test05(self):
        """Verify that the duration of the flow is correct."""

        file = open("../pcapfiles/openflow.pcap", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # The pcap file contains 3 flows
        # [192.168.2.4:45203]:6:[192.168.2.14:6633]
        # [192.168.2.4:46926]:6:[192.168.2.14:22]
        # [192.168.2.8:1044]:17:[239.255.255.250:8082]
        headers = {"Accept" : "application/json"}

        with requests.Session() as session:
            flowid = "[192.168.2.4:45203]:6:[192.168.2.14:6633]"
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("duration", jdata)
            self.assertEqual(jdata["duration"], 7)

            flowid = "[192.168.2.4:46926]:6:[192.168.2.14:22]"
            params = {"id": flowid}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("duration", jdata)
            self.assertEqual(jdata["duration"], 1)

            flowid = "[192.168.2.8:1044]:17:[239.255.255.250:8082]"
            params = {"id": flowid}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("duration", jdata)
            self.assertEqual(jdata["duration"], 1)

    def test06(self):
        """Verify create a new Domain that matchs, generates
        extra information on the DNS"""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python"}
            code = "dom = pyaiengine.DomainName(\"Ubuntu\", \".ubuntu.com\")\n"
            code += "dm = pyaiengine.DomainNameManager([dom])\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Python code loaded successfully", res.text)

            file = open("../pcapfiles/openflow_dns.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            flowid = "[129.21.3.17:53]:17:[192.168.2.6:28848]"
            params = {"id": flowid}
            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("[91.189.92.55, 91.189.92.57]", res.text)
            self.assertIn("Anomaly:          UDP bogus header", res.text)

            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertIn("dns", data)
            self.assertIn("ips", data["dns"])
            self.assertIn("91.189.92.55", data["dns"]["ips"])
            self.assertIn("91.189.92.57", data["dns"]["ips"])

    def test07(self):
        """Verify that we can retrieve DNS map information."""

        with requests.Session() as session:
            # Now inject the pcap file
            file = open("../pcapfiles/openflow_dns.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Check the default output of the DNS protocol
            headers = {"Content-Type" : "text/plain"}
            uri = f"{self.baseuri}/v1/protocol"
            params = {"name": "DNS", "stats_level": 5}
            res = session.get(uri, params=params, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DNS Domains usage", res.text)
            self.assertIn("Domain:daisy.ubuntu.com:3", res.text)

            params = {"name": "DNS", "stats_level": 5, "limit": 1}
            res = session.get(uri, params=params, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DNS Domains usage", res.text)
            self.assertIn("Domain:daisy.ubuntu.com:3", res.text)

            params = {"name": "DNS", "stats_level": 5, "limit": 0}
            res = session.get(uri, params=params, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("DNS Domains usage", res.text)
            self.assertNotIn("Domain:daisy.ubuntu.com:3", res.text)

    def test08(self):
        """Verify that the user can see the L7 payloads of
        a TCP flow."""

        with requests.Session() as session, \
            open("../pcapfiles/openflow_dns.pcap", "rb") as file:

            flowid = "[192.168.1.2:57328]:6:[192.168.1.1:6633]"
            params = {"id": flowid, "payload": True}
            headers = {"Accept": "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            # Now inject the pcap file
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid, "payload": True}
            headers = {"Accept": "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("daisy", res.text)

            params = {"id": flowid, "payload": True, "len": 32}
            headers = {"Accept": "text/plain"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertNotIn("daisy", res.text)

            params = {"id": flowid, "payload": True, "len": 32}
            headers = {"Accept": "application/json"}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertIn("l7_payload", flow)
            self.assertEqual(len(flow["l7_payload"]), 32)


class StackVirtualHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with a network stack for Virtual data centers and
    with an authorization_code_token on the PacketDispatcher."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        cls.token = "".join(random.choice(string.ascii_uppercase) for i in range(16))
        generate_service_code("server_code.py", cls.__name__, "StackVirtual", cls.port,
            "../pcapfiles/4udppackets.pcap",
            extra_dispatcher = [f"pdis.authorization_code_token = \"{cls.token}\""])

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify that we can not execute code on the instance."""

        headers = {"Content-Type" : "text/python"}
        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 401)

    def test02(self):
        """Verify that Basic an Bearer token works."""

        headers = {"Content-Type" : "text/python",
                   "Authorization": "Basic Bad token"}
        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 403)

        headers = {"Content-Type" : "text/python",
                   "Authorization": "Basic %s" % self.token}
        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 200)

        headers = {"Content-Type" : "text/python",
                   "Authorization": "Bearer %s" % self.token}
        code = "a = 1 + 5" + "\n" + "print(a)"
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers)
        self.assertEqual(res.status_code, 200)

    def test03(self):
        """Verify that we can change the token over API."""

        with requests.Session() as session:
            token = "".join(random.choice(string.ascii_uppercase) for i in range(16))
            headers = {"Content-Type" : "text/python",
                       "Authorization": f"Basic {self.token}"}
            code = f"pdis.authorization_code_token = \"{token}\""
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            # With the old token dont works
            headers = {"Content-Type" : "text/python",
                       "Authorization": f"Basic {self.token}"}
            code = f"pdis.authorization_code_token = \"{token}\""
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 403)

            # With the new token works 
            headers = {"Content-Type" : "text/python",
                       "Authorization": "Basic %s" % token}
            code = "print(\"hello\")"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

    def test04(self):
        """Verify that we can empty the authorization code token."""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python",
                       "Authorization": f"Basic {self.token}"}
            code = "pdis.authorization_code_token = \"\""
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            headers = {"Content-Type" : "text/python"}
            code = "print(\"hello\")"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

    def test05(self):
        """Check that no pcap file can be upload without token."""

        with requests.Session() as session:
            files = {"file" : "nothing"}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 401)

            files = {"file" : "nothing"}
            headers = {"Authorization": "Basic %sA" % self.token}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 403)

            files = {"file" : "nothing"}
            headers = {"Authorization": "Basic %s" % self.token[0]}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 403)

            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file" : file}
            headers = {"Authorization": "Basic %s" % self.token}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

    def test06(self):
        """Verify that enable/disable the interpreter of the running
        instance writes the correct logs."""

        with requests.Session() as session:
            headers = {"Content-Type" : "text/python",
                       "Authorization": "Basic %s" % self.token}
            code = "pdis.enable_shell = True"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            uri = f"{self.baseuri}/v1/logs"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Shell enable on %s" % os.ttyname(sys.stdout.fileno()), res.text)

            # The commands should be logged
            headers = {"Content-Type" : "text/python",
                       "Authorization": "Basic %s" % self.token}
            code = "pdis.enable_shell = False"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            uri = f"{self.baseuri}/v1/logs"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Shell disable on %s" % os.ttyname(sys.stdout.fileno()), res.text)

    def test07(self):
        """Verify that by accessing the URI python_code we can see
        the current script code."""

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/current_code"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertTrue(res.text.startswith("# code generated"))

    def test08(self):
        """Verify the URI PUT /v1/flow/<flowid> with parameter for update the flow

        [192.168.100.1:43812]:6:[192.168.100.2:80]
        [192.168.1.79:35819]:17:[192.168.1.20:8472]
        """

        flowid_udp = "[192.168.1.79:35819]:17:[192.168.1.20:8472]"
        flowid_tcp = "[192.168.100.1:43812]:6:[192.168.100.2:80]"
        params = {"id": flowid_udp}

        headers = {}

        with requests.Session() as session:
            file = open("./protocols/vxlan/packets/packet07.pcap", "rb")
            files = {"file" : file}
            headers = {"Authorization": "Basic %s" % self.token}

            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files, headers=headers)
            self.assertEqual(res.status_code, 200)
            file.close()

            data = {"label" : "Flow UDP",
                    "reject" : True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 401)

            # Now put the corrrect Authorization header
            uri = f"{self.baseuri}/v1/flow"
            res = session.put(uri, json=data, params=params, headers=headers)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid_tcp, "tag": 42}
            data = {"label" : "Flow TCP",
                    "reject" : True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 401)

            # Now put the corrrect Authorization header
            uri = f"{self.baseuri}/v1/flow"
            res = session.put(uri, json=data, params=params, headers=headers)
            self.assertEqual(res.status_code, 200)

    def test09(self):
        """Verify the URI PUT /v1/flow/<flowid> with wrong
        parameters on the URI."""

        """
        [192.168.100.1:43812]:6:[192.168.100.2:80]
        [192.168.1.79:35819]:17:[192.168.1.20:8472]
        """

        bad = [
          "[Bububuu////---:43812]:6:[192.168.100.2:80]",
          "[192.168.100.1:xxxxx122243812]:6:[192.168.100.2:80]",
          "[192.168.100.1:43812]:asdfe6:[192.168.100.2:80]",
          "[192.168.100.1:43812]:6:[192.168.100.XXXXXXXX2:80]",
          "[192.168.100.1:43812]:6:[192.168.100.XXXXXXXX2]",
          "[192.168.100.1]:6:[192.168.100.2]",
          "[192.168.100.1]:6:[192.168.",
          "[192.168.100.16:[192.168.",
          "AAAAAAAAAAAAAAAAAAAAAAAAA:BBBBBBBBB",
          "[192.168.100.16192.168.",
          "192.168.100.16192.[168.]",
          "[]:6:[192.168.100.XXXXXXXX2]",
          ""
        ]
        headers = {}

        with requests.Session() as session:
            file = open("./protocols/vxlan/packets/packet07.pcap", "rb")
            files = {"file" : file}
            headers = {"Authorization": "Basic %s" % self.token}

            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files, headers=headers)
            self.assertEqual(res.status_code, 200)
            file.close()

            uri = f"{self.baseuri}/v1/flow"
            for item in bad:
                params = {"id": item}
                res = session.get(uri, params=params)
                self.assertEqual(res.status_code, 404)

            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)

            flowid = "[192.168.1.79:35819]:17:[192.168.1.20:8472]"
            params = {"id": flowid}
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)

    def test10(self):
        """Verify that we can modify a property of an object
        if we have the Authorization header."""

        uri = f"{self.baseuri}/v1/locals"

        with requests.Session() as session:
            headers = {"Accept" : "application/json"}
            params = {"name": "stack", "property": "mode", "value": "nids"}
            # Now change the value of the property
            res = session.put(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 401)

    def test11(self):
        """Verify that we can see the DHCP maps correctly."""

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/pcapfile"
            file = open("../pcapfiles/gre_dhcp.pcap", "rb")
            files = {"file" : file}
            headers = {"Authorization": "Basic %s" % self.token}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Now check the DHCP map
            params = {"name": "dhcp", "stats_level": 5}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Host names usage", res.text)
            self.assertIn("Host:PAQUITO:1", res.text)
            self.assertNotIn("...", res.text)

            params = {"name": "dhcp", "stats_level": 5, "limit": 0}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Host names usage", res.text)
            self.assertIn("...", res.text)
            self.assertNotIn("Host:PAQUITO:1", res.text)

            params = {"name": "dhcp", "stats_level": 5, "limit": 1000}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Host names usage", res.text)
            self.assertIn("Host:PAQUITO:1", res.text)
            self.assertNotIn("...", res.text)

    def test12(self):
        """Verify that we can see the L7 payload of the flow."""

        with requests.Session() as session, \
            open("../pcapfiles/vxlan_ftp.pcap", "rb") as file:

            files = {"file" : file}
            headers = {"Authorization": "Basic %s" % self.token}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)

            flowid = "[192.168.1.2:41024]:17:[192.168.1.1:4789]"
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)

            flowid = "[192.168.1.23:20]:6:[192.168.1.100:1034]"
            params = {"id": flowid, "tag": 1, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)

            fd = open("../pcapfiles/vxlan_ftp.pcap", "rb")
            files = {"file" : fd}
            headers = {"Authorization": "Basic %s" % self.token}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)
            fd.close()

            flowid = "[192.168.1.23:20]:6:[192.168.1.100:1034]"
            params = {"id": flowid, "tag": 1, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("dev", res.text)
            self.assertIn("etc", res.text)
            self.assertIn("lib", res.text)
            self.assertIn("incoming", res.text)
            self.assertIn("pub", res.text)
            self.assertIn("html", res.text)


class StackLanHTTPServerEvidencesTests(unittest.TestCase):
    """ Verify the functionality of the http interface with
        the evidences functionality """

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            "../pcapfiles/vxlan_ftp.pcap",
            extra_dispatcher = [f"pdis.evidences = True"],
            extra_body = ["def http_handler(flow):\n    flow.evidence = True",
                          "stack.set_anomaly_callback(http_handler, \"http\")"])

    def get_evidence_filename(self):
        """Returns the list of evidence files."""

        files = glob.glob("evidences.%d.*.pcap" % self.pid)
        for file in files:
            return file
        return None

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify the URI PUT /v1/flow?id= with parameter for update the flow """

        flowid = "[192.168.1.1:63139]:17:[192.168.1.254:53]"
        params = {"id": flowid}
        headers = {}

        file = open("./protocols/dns/packets/packet08.pcap", "rb")
        files = {"file" : file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # Get the flows in JSON format
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/flows"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(len(res.json()), 1)

        # Get the flow in JSON format
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/flow"
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertNotIn("label", jres)
        self.assertEqual(jres["evidence"], False)

        # Modify the evidence of the flow that is on memory
        data = {"evidence" : True}
        res = requests.put(uri, json=data, params=params)
        self.assertEqual(res.status_code, 200)

        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertEqual(jres["evidence"], True)

        # Now inject again the flow and check the size of the evidences file

        headers = {}

        file = open("./protocols/dns/packets/packet08.pcap", "rb")
        files = {"file" : file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

    def test02(self):
        """Verify that the evidences uri is available and when the user
        will click it download the pcapfile."""

        pcap_filename = self.__class__.__name__.lower() + ".test02.pcap"
        if os.path.exists(pcap_filename):
            os.remove(pcap_filename)

        headers = {}
        with requests.Session() as session:
            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            flowid = "[192.168.1.1:63139]:17:[192.168.1.254:53]"
            params = {"id": flowid}
            # Modify the evidence of the flow that is on memory
            uri = f"{self.baseuri}/v1/flow"
            data = {"evidence" : True}
            res = session.put(uri, params=params, json=data)
            self.assertEqual(res.status_code, 200)

            # Now inject the file again
            file = open("./protocols/dns/packets/packet08.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Download the pcap file
            uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.headers["Content-Type"], "application/vnd.tcpdump.pcap")
            with open(pcap_filename, "wb") as f:
                f.write(res.content)

            # Now we inject the generated pcap file
            file = open(pcap_filename, "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Now we check that the flow should have 3 packets on it
            headers = {"Accept" : "application/json"}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertEqual(jdata["upstream"]["packets"], 3)

    def test03(self):
        """Verify that the evidences works with an HTTP
        anomaly that is set on a callback."""

        pcap_filename = self.__class__.__name__.lower() + ".test03.pcap"
        if os.path.exists(pcap_filename):
            os.remove(pcap_filename)

        headers = {}
        with requests.Session() as session:
            file = open("./protocols/http/packets/packet04.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Download the pcap file
            uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.headers["Content-Type"], "application/vnd.tcpdump.pcap")
            with open(pcap_filename, "wb") as f:
                f.write(res.content)
                size1 = len(res.content)

            file = open(pcap_filename, "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Download the pcap file again and verify that grows
            uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.headers["Content-Type"], "application/vnd.tcpdump.pcap")
            with open(pcap_filename, "wb") as f:
                f.write(res.content)
                size2 = len(res.content)

            self.assertGreater(size2, size1)

    def test04(self):
        """Verify that when the evidences file has been removed
        the file is not accesible by the API."""

        headers = {}
        with requests.Session() as session:
            file = open("./protocols/http/packets/packet04.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Remove all pcap files of the directory including the one
            # that is been used by the EvidenceManager
            files = glob.glob("evidences.*.pcap")
            for pcapfile in files:
                os.remove(pcapfile)

            # Try to download the pcap file
            uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 404)
            self.assertEqual(res.headers["Content-Type"], "text/html")
            self.assertIn("Evidences pcapfile not found", res.text)
            self.assertIn("3000", res.text)

    def test05(self):
        """Verify that fragmented traffic in IPv4 is not been record
        on the generated pcap file."""

        pcap_filename = self.__class__.__name__.lower() + ".test05.pcap"
        if os.path.exists(pcap_filename):
            os.remove(pcap_filename)

        with requests.Session() as session:
            file = open("./protocols/ip/packets/packet04.pcap", "rb")
            files = {"file" : file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            headers = {"Accept": "application/json"}
            uri = f"{self.baseuri}/v1/anomalies"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertIn("anomalies", jdata)
            self.assertIn("IPv4 Fragmentation", jdata["anomalies"][0]["name"])
            self.assertEqual(jdata["anomalies"][0]["hits"], 1)

            # Download the pcap file
            uri = f"{self.baseuri}/v1/evidences.{self.proc.pid}.pcap"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertEqual(res.headers["Content-Type"], "application/vnd.tcpdump.pcap")
            with open(pcap_filename, "wb") as f:
                f.write(res.content)
                size1 = len(res.content)

            self.assertEqual(size1, 24)

            # Remove all pcap files of the directory including the one
            # that is been used by the EvidenceManager
            files = glob.glob("evidences.*.pcap")
            for pcapfile in files:
                os.remove(pcapfile)


class StackLanWithFrequenciesHTTPServerTests(unittest.TestCase):
    """ Verify that the http interface is fine for get information
        but with the feature of enable the frequency analyzer """

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"

    def __service(self):

        stack = pyaiengine.StackLan()

        stack.tcp_flows = 2048
        stack.udp_flows = 1024
        stack.mode = "frequency"

        with pyaiengine.PacketDispatcher("../pcapfiles/4udppackets.pcap") as pdis:
            pdis.stack = stack
            pdis.authorized_ip_address = AUTHORIZED_HOSTS
            pdis.http_port = self.port
            pdis.run()

        sys.exit(0)

    def setUp(self):

        clean_log_files()
        pid = os.fork()
        if pid == 0:
            self.__service()
        else:
            self.pid = pid
            time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        os.kill(self.pid, 9)
        _ = os.wait()

    def test01(self):
        """Upload a pcap file and analyze with frequencies."""

        headers = {}
        file = open("../pcapfiles/dcerpc_traffic.pcapng", "rb")
        files = {"file": file}
        uri = f"{self.baseuri}/v1/pcapfile"
        res = requests.post(uri, files=files)
        self.assertEqual(res.status_code, 200)
        file.close()

        # Get the flows in JSON format
        headers = {"Accept" : "application/json"}
        uri = f"{self.baseuri}/v1/flows"
        res = requests.get(uri, headers=headers)
        self.assertEqual(res.status_code, 200)

        jres = res.json()
        self.assertGreater(len(jres), 12)

        for flow in jres:
            self.assertIn("dispersion", flow)
            self.assertIn("enthropy", flow)
            self.assertIn("frequencies", flow)

    def test02(self):
        """Upload a ssh pcap file and analyze with frequencies."""

        with requests.Session() as session:
            headers = {}

            file = open("../pcapfiles/ssh_flow.pcap", "rb")
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)
            file.close()

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)

            jres = res.json()

            flow = [flow for flow in jres][0]

            self.assertNotEqual(flow, None)
            self.assertGreater(flow["dispersion"], 240)
            self.assertEqual(flow["enthropy"], 0.0)

            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:6:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Network flow", res.text)
            self.assertIn("Protocol:                      TCP", res.text)
            self.assertIn("L7Protocol:           TCPFrequency", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("IP ttl:                   (123,64)", res.text)


class StackLanFifoTests(unittest.TestCase):
    """Verify that the engine is able to read packets
    from a fifo file."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        cls.fifo = "/tmp/testfifo.fifo"
        if not os.path.exists(cls.fifo):
            os.mkfifo(cls.fifo)

    def __service(self):

        stack = pyaiengine.StackLan()

        stack.tcp_flows = 2048
        stack.udp_flows = 1024

        with pyaiengine.PacketDispatcher(self.fifo) as pdis:
            pdis.stack = stack
            pdis.authorized_ip_address = AUTHORIZED_HOSTS
            pdis.http_address = HTTP_LISTEN_IP
            pdis.http_port = self.port
            pdis.run()

        sys.exit(0)

    def setUp(self):

        pid = os.fork()
        if pid == 0:
            self.__service()
        else:
            self.pid = pid
            time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        os.kill(self.pid, 9)
        _ = os.wait()

    def test01(self):
        """Inject a pcap file on the fifo file."""

        pcapfile = "../pcapfiles/dcerpc_traffic.pcapng"
        cmd = f"tcpdump -s 0 -n -w - -U -r {pcapfile} > {self.fifo}"
        os.system(cmd)
        time.sleep(API_DELAY)

        with requests.Session() as session:
            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            retries = Retry(connect=5)

            session.mount("http://", HTTPAdapter(max_retries=retries))
            res = session.get(uri, headers=headers)

            self.assertIsNotNone(res)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            self.assertGreater(len(jres), 12)


@unittest.skipIf(os.geteuid() != 0, "Test not supported")
class StackLanNetworkDeviceStatusCallbackTests(unittest.TestCase):
    """Verify that the engine is able to receive network
    device changes over netlink."""

    def __service(self):

        def network_device_callback():
            # Function for process the network device events
            if pdis.device_status == "up":
                open(self.filename_up, "a").close()
            elif pdis.device_status == "down":
                open(self.filename_down, "a").close()

        stack = pyaiengine.StackLan()

        stack.tcp_flows = 2048
        stack.udp_flows = 1024

        with pyaiengine.PacketDispatcher(self.network_device) as pdis:
            pdis.stack = stack
            pdis.network_device_change_callback = network_device_callback
            pdis.run()

        sys.exit(0)

    def setUp(self):

        self.network_device = os.environ.get("NETWORK_DEVICE", "lo")
        self.filename_up = f"device_{self.network_device}_up.txt"
        self.filename_down = f"device_{self.network_device}_down.txt"

        if os.path.exists(self.filename_up):
            os.remove(self.filename_up)

        if os.path.exists(self.filename_down):
            os.remove(self.filename_down)

        pid = os.fork()
        if pid == 0:
            self.__service()
        else:
            self.pid = pid
            time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        os.kill(self.pid, 9)
        _ = os.wait()

    def test01(self):
        """Check that the PacketDispatcher triggers the callbacks
        when there are changes on network interfaces."""

        os.system(f"ifconfig {self.network_device} down")
        time.sleep(1)
        self.assertTrue(os.path.exists(self.filename_down))
        os.system(f"ifconfig {self.network_device} up")
        time.sleep(1)
        self.assertTrue(os.path.exists(self.filename_up))


@unittest.skipIf((os.geteuid() != 0 or which("tcpreplay") == None), "Test not supported")
class StackLanLiveHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with live traffic capture from a network interface.
    On linux will be lo and on macos en0."""

    @classmethod
    def setUpClass(cls):
        cls.interface = "lo" if sys.platform == "linux" else "en0"
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            f"{cls.interface}")

    def setUp(self):
        """Create a new process with the server code"""
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Test the that ssl traffic is correct."""

        pcapfile = "../pcapfiles/sslflow.pcap"
        cmd = f"%s -t -i {self.interface} {pcapfile} 2>/dev/null 1>&2" % which("tcpreplay")
        os.system(cmd)

        uri = f"{self.baseuri}/v1/flow"
        params = {"id": "[192.168.1.13:44265]:6:[74.125.24.189:443]"}
        headers = {"Accept" : "application/json"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        data = res.json()
        self.assertEqual(data["layer7"], "SSL")
        value = hex(data["ssl"]["cipher"])
        self.assertEqual(value, "0xc011")
        self.assertEqual(data["ssl"]["host"], "0.drive.google.com")
        self.assertEqual(data["ssl"]["issuer"], "Google Internet Authority")

        uri = f"{self.baseuri}/v1/flow"
        params = {"id": "[192.168.1.13:44265]:6:[74.125.24.189:443]"}
        headers = {"Accept" : "text/plain"}
        res = requests.get(uri, headers=headers, params=params)
        self.assertEqual(res.status_code, 200)
        self.assertIn("TCP: Flg[S(1)SA(1)", res.text)
        self.assertIn("Seq(362328793,3500511283)WS(661,1002)]", res.text)
        self.assertIn("Layer7: Pdus:45 Ver:0x302 Cipher:0xc011 Host:0.drive.google.com Issuer:Google Internet Authority", res.text)

    def test02(self):
        """Test the that ssl traffic is correct with pop3"""

        pcapfile = "../pcapfiles/pop3_starttls.pcap"
        cmd = f"%s -t -L 30 -i {self.interface} {pcapfile} 2>/dev/null 1>&2" % which("tcpreplay")
        os.system(cmd)

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertGreater(data["process packets"], 13)

            params = {"id": "[192.168.4.149:54775]:6:[192.168.4.149:110]"}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["layer7"], "SSL")
            value = hex(data["ssl"]["cipher"])
            self.assertEqual(value, "0x9f")
            self.assertEqual(data["ssl"]["issuer"], "Lilawelt")
            self.assertEqual(data["ssl"]["mtls"], True)

            params = {"name": "ssl", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            res = session.get(uri, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Total client hellos:             1", res.text)
            self.assertIn("Total server hellos:             1", res.text)
            self.assertIn("Total certificates:              2", res.text)

    def test03(self):
        """Test the that ssl traffic is correct"""

        pcapfile = "../pcapfiles/alibaba.pcap"
        cmd = f"%s -t -i {self.interface} {pcapfile} 2>/dev/null 1>&2" % which("tcpreplay")
        os.system(cmd)

        with requests.Session() as session:
            params = {"id": "[10.42.0.211:51057]:6:[64.71.142.82:443]"}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow1 = res.json()
            self.assertEqual(flow1["layer7"], "SSL")

            params = {"id": "[10.42.0.211:38380]:6:[64.71.142.110:443]"}
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow2 = res.json()
            self.assertEqual(flow2["layer7"], "SSL")
            self.assertEqual(flow1["ssl"]["cipher"], flow2["ssl"]["cipher"])
            self.assertEqual(flow1["ssl"]["host"], flow2["ssl"]["host"])
            self.assertEqual(flow1["ssl"]["session"], flow2["ssl"]["session"])

    def test04(self):
        """Test that we can close the PacketDispatcher and having
        the webserver up and running and restart capturing traffic."""

        pcapfile = "../pcapfiles/alibaba.pcap"
        cmd = f"%s -t -i {self.interface} {pcapfile} 2>/dev/null 1>&2" % which("tcpreplay")

        with requests.Session() as session:
            # The PacketDispatcher is up and running reading from an interface
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("device", res.json())

            code = "pdis.close()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Python code loaded successfully", res.text)

            # Inject the traffic and nothing should be capture
            os.system(cmd)

            # The interface is not available
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertNotIn("device", res.json())

            flows = ["[10.42.0.211:51057]:6:[64.71.142.82:443]",
                     "[10.42.0.211:38380]:6:[64.71.142.110:443]"]

            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept" : "application/json"}
            for flow in flows:
                params = {"id": flow}
                res = session.get(uri, headers=headers, params=params)
                self.assertEqual(res.status_code, 404)

            # Now we capture again from the device
            code = f"pdis.open(\"{self.interface}\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        # Inject the traffic that should be capture
        os.system(cmd)

        with requests.Session() as session:

            flows = ["[10.42.0.211:51057]:6:[64.71.142.82:443]",
                     "[10.42.0.211:38380]:6:[64.71.142.110:443]"]

            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept" : "application/json"}
            for flow in flows:
                params = {"id": flow}
                res = session.get(uri, headers=headers, params=params)
                self.assertEqual(res.status_code, 200)

    def test05(self):
        """Test that the engine shows on the logs that using an IP
        front the listenting network device is not a good idea."""

        pcapfile = "../pcapfiles/alibaba.pcap"
        cmd = f"%s -t -i {self.interface} {pcapfile} 2>/dev/null 1>&2" % which("tcpreplay")

        with requests.Session() as session:
            # The PacketDispatcher is up and running reading from an interface
            uri = f"{self.baseuri}/v1/logs"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("[warning] Management IP '127.0.0.1' belongs to network device 'lo'", res.text)


@unittest.skipIf((sys.platform != "linux" or os.geteuid() != 0 or which("tcpreplay") == None), "Test not supported")
class StackLanNetfilterLiveHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with live traffic capture from a netfilter queue on Linux works."""

    @classmethod
    def setUpClass(cls):
        cls.interface = "netfilter1"
        cls.port = get_free_port()
        cls.ipport = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            f"{cls.interface}")

    def setUp(self):
        """Create a new process with the server code"""

        os.system(f"iptables -I INPUT -i lo -p tcp --dport {self.ipport} -j NFQUEUE --queue-num 1")
        os.system(f"iptables -I INPUT -i lo -p udp --dport {self.ipport} -j NFQUEUE --queue-num 1")
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()
        os.system(f"iptables -D INPUT -i lo -p tcp --dport {self.ipport} -j NFQUEUE --queue-num 1")
        os.system(f"iptables -D INPUT -i lo -p udp --dport {self.ipport} -j NFQUEUE --queue-num 1")

    def test01(self):
        """Verify that SYN packets are been seen by the PacketDispatcher."""

        try:
            res = requests.get(f"http://localhost:{self.ipport}", timeout=4)
        except Exception as exc:
            pass

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Total accept packets:            1", res.text)
            self.assertIn("Total drop packets:              0", res.text)
            self.assertIn("Total accept bytes:             74", res.text)
            self.assertIn("Total drop bytes:                0", res.text)

            params = {"name": "tcp", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["flags"]["acks"], 0)
            self.assertEqual(data["flags"]["fins"], 0)
            self.assertEqual(data["flags"]["rsts"], 0)
            self.assertEqual(data["flags"]["synacks"], 0)
            self.assertEqual(data["flags"]["syns"], 1)

    def test02(self):
        """Verify that we can drop by IP address."""

        # Now we drop any traffic from 127.0.0.1
        code = "def drop_callback(flow):\n"
        code += "    flow.accept = False\n"
        code += "ipset = pyaiengine.IPSet(['127.0.0.1'])\n"
        code += "ipset.callback = drop_callback\n"
        code += "ipmg = pyaiengine.IPSetManager()\n"
        code += "ipmg.add_ip_set(ipset)\n"
        code += "stack.tcp_ip_set_manager = ipmg\n"

        headers = {"Content-Type" : "text/python"}
        uri = f"{self.baseuri}/v1/python_code"
        res = requests.post(uri, data=code, headers=headers, timeout=2)
        self.assertEqual(res.status_code, 200)

        try:
            res = requests.get(f"http://localhost:{self.ipport}", timeout=4)
        except:
            pass

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Total accept packets:            0", res.text)
            self.assertIn("Total drop packets:              3", res.text)
            self.assertIn("Total accept bytes:              0", res.text)
            self.assertIn("Total drop bytes:              222", res.text)
 
            # Flow information
            params = {"protocol": 6, "dstport": self.ipport}
            uri = f"{self.baseuri}/v1/flows"
            headers = {"Accept": "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(len(data), 1)

            flow = data[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:6:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Accept:                         no", res.text)
            self.assertIn("TCP state:                SYN_SENT", res.text)
            self.assertIn("TCP: Flg[S(3)SA(0)A(0)F(0)R(0)P(0)", res.text)
            self.assertIn("IPSet:               Generic IPSet", res.text)
            self.assertIn("Drop packets:                    3", res.text)
            self.assertIn("Drop bytes:                      0", res.text)

    def test03(self):
        """Verify that we can drop by regex on UDP."""

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            regex_name ="some regex"
            code = "def drop_callback(flow):\n"
            code += "    flow.accept = False\n"
            code += "    flow.evidence = True\n"
            code += f"regex = pyaiengine.Regex('{regex_name}', 'AAAA')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.udp_regex_manager = remg\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            sock.sendto(b"AAAA", ("127.0.0.1", self.ipport))

            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["accept_bytes"], 0)
            self.assertEqual(data["accept_packets"], 0)
            self.assertEqual(data["bytes"], 46)
            self.assertEqual(data["process packets"], 1)
            self.assertEqual(data["drop_bytes"], 46)
            self.assertEqual(data["drop_packets"], 1)

            sock.sendto(b"dont match", ("127.0.0.1", self.ipport))

            # The flow should be dropped
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["accept_bytes"], 0)
            self.assertEqual(data["accept_packets"], 0)
            self.assertEqual(data["drop_packets"], 2)
            self.assertEqual(data["drop_bytes"], 98)

            params = {"protocol": 17, "dstport": self.ipport}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(len(data), 1)
            flow = data[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            # Get the flow in plain format
            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:             UDPGeneric", res.text)
            self.assertIn("Accept:                         no", res.text)
            self.assertIn("Evidence:                      yes", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Regex:                  some regex", res.text)
            self.assertIn("IP ttl:                     (64,0)", res.text)
            self.assertIn("Drop bytes:                     30", res.text)
            self.assertIn("Drop packets:                    2", res.text)

    def test04(self):
        """Verify that we can drop by DomainName on HTTP."""

        proc_http = subprocess.Popen([sys.executable, "-m", "http.server", f"{self.ipport}"])

        time.sleep(API_DELAY)

        with requests.Session() as session:

            code = "def drop_callback(flow):\n"
            code += "    flow.accept = False\n"
            code += "dom = pyaiengine.DomainName('Some domain', '*')\n"
            code += "dom.callback = drop_callback\n"
            code += "dman = pyaiengine.DomainNameManager()\n"
            code += "dman.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dman, 'http')\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            try:
                res = requests.get(f"http://localhost:{self.ipport}", timeout=4)
            except:
                pass

            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["accept_bytes"], 140)
            self.assertEqual(data["accept_packets"], 2)
            self.assertEqual(data["bytes"], 1478)
            self.assertEqual(data["process packets"], 9)
            self.assertEqual(data["drop_bytes"], 1338)
            self.assertEqual(data["drop_packets"], 7)

        proc_http.kill()
        proc_http.wait()

    def test05(self):
        """Verify that we can switch with different types of sources, in this test
        we change from netfilter -> lo -> netfilter."""

        with requests.Session() as session:
            # Now we capture from lo
            code = f"pdis.close()\npdis.open(\"lo\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["device"], "lo")
            self.assertEqual(data["device_status"], 0)

            # Now we change again to netfilter device
            code = f"pdis.close()\npdis.open(\"netfilter1\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["device"], "netfilter1")
            self.assertEqual(data["device_status"], 0)

    def test06(self):
        """Verify that we can switch with different types of sources, in this test
        we change from netfilter -> pcap file -> netfilter."""

        pcapfile = "../pcapfiles/alibaba.pcap"
        with requests.Session() as session:
            # Now we capture from the pcap file
            code = f"pdis.close()\npdis.open(\"{pcapfile}\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["pcapfile"], pcapfile)

            # Now we change again to netfilter device
            code = f"pdis.close()\npdis.open(\"netfilter1\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        with requests.Session() as session:
            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["device"], "netfilter1")
            self.assertEqual(data["device_status"], 0)

    def test07(self):
        """Verify that we can have evidences of the flows that we drop."""

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            code = "def drop_callback(flow):\n"
            code += "    flow.label = 'Flow dropped and write on disk'\n"
            code += "    flow.evidence = True\n"
            code += "    flow.accept = False\n"
            code += "regex = pyaiengine.Regex('some regex', 'AAAA')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.udp_regex_manager = remg\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            # Enable the evidences on the PacketDispatcher
            code = "pdis.evidences = True\n"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            sock.sendto(b"AAAA", ("127.0.0.1", self.ipport))
            sock.sendto(b"No match but is drop", ("127.0.0.1", self.ipport))

            uri = f"{self.baseuri}/v1/summary"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["device"], "netfilter1")
            self.assertEqual(data["device_status"], 0)
            self.assertIn("evidences", data)
            self.assertEqual(data["evidences"]["packets"], 2)
            self.assertEqual(data["drop_packets"], 2)
            filename = data["evidences"]["current_file"]
            os.remove(filename)

            params = {"protocol": 17, "dstport": self.ipport}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(len(data), 1)
            flow = data[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            self.assertEqual(flow["label"], "Flow dropped and write on disk")
            self.assertEqual(flow["evidence"], True)
            self.assertEqual(flow["accept"], False)

            # Get the flow in plain format
            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:             UDPGeneric", res.text)
            self.assertIn("Accept:                         no", res.text)
            self.assertIn("Evidence:                      yes", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                     active", res.text)
            self.assertIn("Up bytes:                       24", res.text)
            self.assertIn("Down bytes:                      0", res.text)
            self.assertIn("Up packets:                      2", res.text)
            self.assertIn("Down packets:                    0", res.text)
            self.assertIn("Duration:              00:00:00:01", res.text)
            self.assertIn("Last packet seen:           0 secs", res.text)
            self.assertIn("IP ttl:                     (64,0)", res.text)
            self.assertNotIn("TCP state:", res.text)

    def test08(self):
        """Verify that we can have evidences of the flows that we drop
        and also that information is reflected on a DatabaseAdaptor object."""

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            label = " Flow dropped and wirte on disk and on adaptor"

            code = "class DatabaseTestAdaptor(pyaiengine.DatabaseAdaptor, object):\n"
            code += "    def __init__(self):\n"
            code += "        self.data = []\n\n"
            code += "    def update (self, _, data):\n"
            code += "        self.data.append(data)\n"
            code += "    def insert (self, key):\n"
            code += "        pass\n"
            code += "    def remove(self, key):\n"
            code += "        pass\n"
            code += "\n"
            code += "def drop_callback(flow):\n"
            code += f"    flow.label = '{label}'\n"
            code += "    flow.evidence = True\n"
            code += "    flow.accept = False\n"
            code += "regex = pyaiengine.Regex('some regex', 'AAAA')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.udp_regex_manager = remg\n"
            code += "adaptor = DatabaseTestAdaptor()\n"
            code += "pdis.evidences = True\n"
            code += "stack.set_udp_database_adaptor(adaptor, 16)\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/locals"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertIn("objects", jres)
            objects = jres["objects"]
            self.assertEqual(objects["DatabaseTestAdaptor"], "class")
            self.assertEqual(objects["adaptor"], "DatabaseTestAdaptor")
            self.assertEqual(objects["drop_callback"], "function")
            self.assertEqual(objects["regex"], "Regex")
            self.assertEqual(objects["remg"], "RegexManager")

            sock.sendto(b"AAAA", ("127.0.0.1", self.ipport))

            code = "print(adaptor.data)"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)
            self.assertIn("accept", res.text)
            self.assertIn("evidence", res.text)
            self.assertIn(label, res.text)

    def test09(self):
        """Verify that we can switch with different types of sources, in this test
        we change from netfilter -> lo and the traffic flow is been seen
        correctly independently of the source."""

        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

        sock.sendto(b"AAAABBBBCCCCDDDD", ("127.0.0.1", self.ipport))

        flowid = ""
        with requests.Session() as session:

            headers = {"Accept" : "application/json"}
            params = {"protocol": 17, "dstport": self.ipport}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(len(data), 1)
            flow = data[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            # Get the flow in plain format
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["downstream"]["bytes"], 0)
            self.assertEqual(data["downstream"]["packets"], 0)
            self.assertEqual(data["upstream"]["bytes"], 16)
            self.assertEqual(data["upstream"]["packets"], 1)

            # Now we capture from lo
            code = f"pdis.close()\npdis.open(\"lo\")\npdis.run()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            # Re run the PacketDispatcher will leave the existing
            # HTTP session comatose, thats why we timeout the session
            try:
                _ = session.post(uri, data=code, headers=headers, timeout=2)
                self.assertTrue(False)
            except:
                self.assertTrue(True)

        sock.sendto(b"AAAABBBBCCCCDDDD", ("127.0.0.1", self.ipport))

        with requests.Session() as session:
            headers = {"Accept" : "application/json"}

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()

            self.assertEqual(data["downstream"]["bytes"], 0)
            self.assertEqual(data["downstream"]["packets"], 0)
            self.assertEqual(data["upstream"]["bytes"], 32)
            self.assertEqual(data["upstream"]["packets"], 2)

        sock.close()

    def test10(self):
        """Verify that we can see the L7 payloads with netfilter."""

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            sock.sendto(b"AAAABBBBCCCCDDDD", ("127.0.0.1", self.ipport))

            headers = {"Accept" : "application/json"}
            params = {"protocol": 17, "dstport": self.ipport}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(len(data), 1)
            flow = data[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            # Get the flow in plain format
            headers = {"Accept": "text/plain"}
            params = {"id": flowid, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            data = "XXXXXXXXXXXXXXXX"
            sock.sendto(str.encode(data), ("127.0.0.1", self.ipport))

            headers = {"Accept": "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertIn("l7_payload", flow)
            self.assertEqual(len(flow["l7_payload"]), 16)


class StackMobileHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with a network stack for mobile envs and
    with an authorization_code_token on the PacketDispatcher."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        cls.token = "".join(random.choice(string.ascii_uppercase) for i in range(16))
        generate_service_code("server_code.py", cls.__name__, "StackMobile", cls.port,
            "../pcapfiles/flow_vlan_x11.pcap",
            extra_dispatcher = [f"pdis.authorization_code_token = \"{cls.token}\""])

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Upload a SIP pcap file and analyze the flow information."""

        with requests.Session() as session, \
            open("../pcapfiles/gprs_sip_flow.pcap", "rb") as file:

            headers = {"Authorization": f"Basic {self.token}"}
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)

            jres = res.json()
            flow = jres[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            # Get the flow in plain format
            headers = {}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:                   GPRS", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("Up bytes:                    15329", res.text)
            self.assertIn("Down bytes:                      0", res.text)
            self.assertIn("Up packets:                     22", res.text)
            self.assertIn("Down packets:                    0", res.text)
            self.assertIn("IP ttl:                     (64,0)", res.text)
            self.assertIn("Duration:              00:00:00:10", res.text)

            flow = jres[1]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:                    SIP", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("Up bytes:                     6575", res.text)
            self.assertIn("Down bytes:                   7962", res.text)
            self.assertIn("Up packets:                     10", res.text)
            self.assertIn("Down packets:                   12", res.text)
            self.assertIn("IP ttl:                    (64,64)", res.text)
            self.assertIn("Duration:              00:00:00:10", res.text)


    def test02(self):
        """Upload a TCP pcap file and analyze the flow information."""

        with requests.Session() as session, \
            open("../pcapfiles/gprs_ftp.pcap", "rb") as file:

            data = {"someting": "value"}
            label = json.dumps(data)
            code = "def drop_callback(flow):\n"
            code += f"    flow.label = '{label}'\n"
            code += "regex = pyaiengine.Regex('some regex', 'CRap')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.tcp_regex_manager = remg\n"

            headers = {"Content-Type" : "text/python",
                "Authorization": f"Basic {self.token}"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            headers = {"Authorization": f"Basic {self.token}"}
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 3)
 
            flow = jres[2]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:6:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept": "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      TCP", res.text)
            self.assertIn("L7Protocol:             TCPGeneric", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("Up bytes:                     2720", res.text)
            self.assertIn("Down bytes:                      0", res.text)
            self.assertIn("Up packets:                      2", res.text)
            self.assertIn("Down packets:                    1", res.text)
            self.assertIn("Duration:              00:00:00:01", res.text)
            self.assertIn("Regex:                  some regex", res.text)

    def test03(self):
        """Verify that we can access to the GTP layer flow information."""

        with requests.Session() as session, \
            open("./protocols/gprs/packets/packet04.pcap", "rb") as file:

            headers = {"Authorization": f"Basic {self.token}"}
            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, headers=headers, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 1)

            flow = jres[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept": "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Protocol:                      UDP", res.text)
            self.assertIn("L7Protocol:                   GPRS", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:                      None", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("GPRS: IMSI(234308256005467) IPv4", res.text)

class StackLanIPv6HTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface with IPv6."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLanIPv6", cls.port,
            "../pcapfiles/4udppackets.pcap")

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify information of IPv6 flows."""

        with requests.Session() as session, \
            open("./protocols/ip6/packets/packet08.pcap", "rb") as file:

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 1)
            flow = jres[0]
            self.assertEqual(flow["ip"]["dst"], "1001::140")
            self.assertEqual(flow["ip"]["src"], "1001::133")
            self.assertEqual(flow["port"]["src"], 55617)
            self.assertEqual(flow["port"]["dst"], 80)

            flow = jres[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:6:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Anomaly:IPv6 Loop ext headers", res.text)
            self.assertIn(flowid, res.text)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept": "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Network flow: 1001::133:55617:6:1001::140:80", res.text)
            self.assertIn("Protocol:                      TCP", res.text)
            self.assertIn("L7Protocol:                   HTTP", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:     IPv6 Loop ext headers", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("Up bytes:                      155", res.text)
            self.assertIn("Down bytes:                      0", res.text)
            self.assertIn("Up packets:                      1", res.text)
            self.assertIn("Down packets:                    0", res.text)
            self.assertIn("Duration:              00:00:00:01", res.text)

            # Do a put and modify the value of the label and also
            # drop the flow
            data = {"label" : "label my friend",
                    "accept" : False}
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept": "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Network flow: 1001::133:55617:6:1001::140:80", res.text)
            self.assertIn("Accept:                         no", res.text)
            self.assertIn("Label:             label my friend", res.text)

    def test02(self):
        """Verify information of IPv6 flows."""

        with requests.Session() as session, \
            open("./protocols/ip6/packets/packet02.pcap", "rb") as file:

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 1)
            flow = jres[0]
            self.assertEqual(flow["ip"]["dst"], "2001:db8:1::1")
            self.assertEqual(flow["ip"]["src"], "2001:db8:1::2")
            self.assertEqual(flow["port"]["src"], 36951)
            self.assertEqual(flow["port"]["dst"], 80)

            # [2001:db8:1::2:36951]:6:[2001:db8:1::1:80]
            flow = jres[0]
            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:6:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            headers = {"Accept" : "text/plain"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Anomaly:HTTP malformed URI", res.text)
            self.assertIn(flowid, res.text)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            headers = {"Accept": "text/plain"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Network flow: 2001:db8:1::2:36951:6:2001:db8:1::1:80", res.text)
            self.assertIn("Protocol:                      TCP", res.text)
            self.assertIn("L7Protocol:                   HTTP", res.text)
            self.assertIn("Accept:                        yes", res.text)
            self.assertIn("Evidence:                       no", res.text)
            self.assertIn("Reject:                         no", res.text)
            self.assertIn("Anomaly:        HTTP malformed URI", res.text)
            self.assertIn("Status:                   comatose", res.text)
            self.assertIn("Up bytes:                       15", res.text)
            self.assertIn("Down bytes:                      0", res.text)
            self.assertIn("Up packets:                      1", res.text)
            self.assertIn("Down packets:                    0", res.text)
            self.assertIn("Duration:              00:00:00:01", res.text)

    def test03(self):
        """Verify information of IPv6 flow with a DomainNameManager
        that is passed over the API."""

        with requests.Session() as session, \
            open("../pcapfiles/ipv6_mix_traffic.pcap", "rb") as file:

            label = "buuuuu"
            # First we generate a basic dns callback
            headers = {"Content-Type" : "text/python"}
            code = "def callback_dns(flow):\n"
            code += f"    flow.label = '{label}'\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom = pyaiengine.DomainName(\"Domain1\", \".yahoo.com\")\n"
            code += "dom.callback = callback_dns\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"dns\")\n"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the DNS flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"l7protocol": "dns"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 18)
            flow = jres[16]
            self.assertEqual(flow["ip"]["dst"], "3ffe:501:4819::42")
            self.assertEqual(flow["ip"]["src"], "3ffe:507:0:1:200:86ff:fe05:80da")
            self.assertEqual(flow["port"]["src"], 2397)
            self.assertEqual(flow["port"]["dst"], 53)
            self.assertEqual(flow["accept"], True)
            self.assertEqual(flow["evidence"], False)
            self.assertEqual(flow["reject"], False)
            self.assertEqual(flow["label"], label)

            flowid = "[" + flow["ip"]["src"] + ":" + str(flow["port"]["src"]) + "]:17:["
            flowid += flow["ip"]["dst"] + ":" + str(flow["port"]["dst"]) + "]"

            label = "label my friend"
            # Modify some values of the flow
            data = {"label" : label,
                    "accept" : False,
                    "evidence": True,
                    "reject": True}
            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.put(uri, json=data, params=params)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = requests.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["ip"]["dst"], "3ffe:501:4819::42")
            self.assertEqual(flow["ip"]["src"], "3ffe:507:0:1:200:86ff:fe05:80da")
            self.assertEqual(flow["port"]["src"], 2397)
            self.assertEqual(flow["port"]["dst"], 53)
            self.assertEqual(flow["accept"], False)
            self.assertEqual(flow["evidence"], True)
            self.assertEqual(flow["reject"], True)
            self.assertEqual(flow["label"], label)

    def test04(self):
        """Verify information of IPv6 flow with a RegexManager
        that is passed over the API."""

        with requests.Session() as session, \
            open("../pcapfiles/ipv6_bgp.pcapng", "rb") as file:

            # First we generate a basic regex callback
            headers = {"Content-Type" : "text/python"}

            label = "This flow is bad"
            code = "def drop_callback(flow):\n"
            code += f"    flow.label = '{label}'\n"
            code += "    flow.evidence = True\n"
            code += "    flow.accept = False\n"
            code += "regex = pyaiengine.Regex('Some example regex', b'^\\xff\\xff\\xff\\xff')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.tcp_regex_manager = remg\n"

            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"l7protocol": "tcpgeneric"}
            uri = f"{self.baseuri}/v1/flows"
            res = requests.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 1)
            flow = jres[0]
            self.assertEqual(flow["ip"]["dst"], "2003:de:2016:1ff::1")
            self.assertEqual(flow["ip"]["src"], "2003:de:2016:1ff::14")
            self.assertEqual(flow["port"]["src"], 65513)
            self.assertEqual(flow["port"]["dst"], 179)
            self.assertEqual(flow["accept"], False)
            self.assertEqual(flow["evidence"], True)
            self.assertEqual(flow["label"], label)
            self.assertEqual(flow["matchs"], "Some example regex")


class StackMobileIPv6HTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface with IPv6."""

    @classmethod
    def setUpClass(cls):
        cls.port = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackMobileIPv6", cls.port,
            "../pcapfiles/gprs_ip6_udp.pcap")

    def setUp(self):
        """Create a new process with the server code"""
        clean_log_files()
        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify information of IPv4 and IPv6 flows."""

        with requests.Session() as session:

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(len(jres), 2)
            flow = jres[0]

            self.assertEqual(flow["ip"]["dst"], "127.0.1.100")
            self.assertEqual(flow["ip"]["src"], "127.0.1.1")
            self.assertEqual(flow["port"]["src"], 2152)
            self.assertEqual(flow["port"]["dst"], 2152)
            self.assertEqual(flow["name"], "GPRS")
            self.assertEqual(flow["layer7"], "GPRS")

            flow = jres[1]

            self.assertEqual(flow["ip"]["dst"], "fd01::183")
            self.assertEqual(flow["ip"]["src"], "fd00:183:1:1:1886:9040:8605:32b8")
            self.assertEqual(flow["port"]["src"], 5060)
            self.assertEqual(flow["port"]["dst"], 5060)
            self.assertEqual(flow["name"], "SIP")
            self.assertEqual(flow["layer7"], "SIP")


    def test02(self):
        """Verify that we can see the l7_payload of the low flow."""

        flowid = "[127.0.1.1:2152]:17:[127.0.1.100:2152]"

        with requests.Session() as session, \
            open("../pcapfiles/gprs_ip6_udp.pcap", "rb") as file:

            # The user marks the flow with payload=True
            headers = {"Accept" : "text/plain"}
            params = {"id": flowid, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            # self.assertNotIn("SIP/2.0", res.text)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "text/plain"}
            params = {"id": flowid, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("SIP/2.0", res.text)

    def test03(self):
        """Verify that we can see the l7_payload of the high flow."""

        flowid = "[fd00:183:1:1:1886:9040:8605:32b8:5060]:17:[fd01::183:5060]"

        with requests.Session() as session, \
            open("../pcapfiles/gprs_ip6_udp.pcap", "rb") as file:

            # The user marks the flow with payload=True
            headers = {"Accept" : "text/plain"}
            params = {"id": flowid, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertNotIn("INVITE", res.text)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            headers = {"Accept" : "text/plain"}
            params = {"id": flowid, "payload": True}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("INVITE", res.text)

    def test04(self):
        """Verify that we can log all the python commands that the user sents
        over the Rest API for log purposes."""

        with requests.Session() as session:

            headers = {"Content-Type" : "text/python"}
            code = "pdis.log_rest_api_code = True"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            items = glob.glob(f"/tmp/aiengine-private-{self.proc.pid}/*.py")
            self.assertFalse(items)

            code = "a = 1 + 5" + "\n" + "print(a)"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            items = glob.glob(f"/tmp/aiengine-private-{self.proc.pid}/*.py")
            self.assertTrue(items)
            self.assertEqual(len(items), 1)

            with open(items[0], "r") as fd_py:
                data = fd_py.read()
                self.assertIn(code, data)

            time.sleep(API_DELAY)

            code = "pdis.log_rest_api_code = False"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            items = glob.glob(f"/tmp/aiengine-private-{self.proc.pid}/*.py")
            self.assertEqual(len(items), 2)

            time.sleep(API_DELAY)

            code = "a = 1 + 5" + "\n" + "print(a)"
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            items = glob.glob(f"/tmp/aiengine-private-{self.proc.pid}/*.py")
            self.assertEqual(len(items), 2)

    @unittest.skipIf(not defined("HAVE_IPCS_QUEUE"), "Test not supported")
    def test05(self):
        """Verify that we can forward alarms to a web service."""

        port = get_free_port()
        proc_http = subprocess.Popen([sys.executable, "-m", "http.server", f"{port}"])

        time.sleep(API_DELAY)

        with requests.Session() as session, \
            open("../pcapfiles/gprs_ip6_tcp.pcap", "rb") as file:

            code = "def drop_callback(flow):\n"
            code += "    flow.alerted = True\n"
            code += "    flow.accept = False\n"
            code += "regex = pyaiengine.Regex('some regex', 'REGISTER')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.tcp_regex_manager = remg\n"
            code += "stack.disable_protocol('sip')\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            time.sleep(QUEUE_DELAY)
            cmd = ["./alarmreader", "-t", "1", "-H", f"{HTTP_LISTEN_IP}", "-p", f"{port}", "-u", "/v1/alarms"]
            ret = subprocess.run(cmd)
            self.assertEqual(ret.returncode, 0)
            proc_http.kill()
            proc_http.wait()

    def test06(self):
        """Verify that we can make external API calls on callbacks. 
        This is not recomended because the engine will be blocked on procesing
        packets but sometimes could be helpfull."""

        port = get_free_port()
        proc_http = subprocess.Popen([sys.executable, "-m", "http.server", f"{port}"])

        time.sleep(API_DELAY)

        with requests.Session() as session, \
            open("../pcapfiles/gprs_ip6_tcp.pcap", "rb") as file:

            url = f"http://localhost:{port}"
            code = "def drop_callback(flow):\n"
            code += "    import requests\n"
            code += f"    res = requests.get(\"{url}\")\n"
            code += "regex = pyaiengine.Regex('some regex', 'REGISTER')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.tcp_regex_manager = remg\n"
            code += "stack.disable_protocol('sip')\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            files = {"file": file}
            uri = f"{self.baseuri}/v1/pcapfile"
            res = session.post(uri, files=files)
            self.assertEqual(res.status_code, 200)

            time.sleep(QUEUE_DELAY)
            proc_http.kill()
            proc_http.wait()


@unittest.skipIf((sys.platform != "linux" or os.geteuid() != 0 or which("tcpreplay") == None), "Test not supported")
class StackLanNetfilterDynamicLiveHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with live traffic capture from a localhost and then switch
    under certain conditions to drop the traffic with netfilter dynamically."""

    @classmethod
    def setUpClass(cls):
        cls.interface = "lo"
        cls.port = get_free_port()
        cls.ipport = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            f"{cls.interface}")

    def setUp(self):
        """Create a new process with the server code"""

        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Verify that a the engine can change to netfilter mode
        on a callback and verify that traffic is been dropped."""

        port = get_free_port()

        proc_http = Process(target=http_server, args=(port,))
        proc_http.daemon = True
        proc_http.start()

        time.sleep(API_DELAY)

        with requests.Session() as session:

            # Set a callback that when a flow matchs it will create
            # an iptables rule dynamically and the engine will start
            # to drop the packets as the packet dispatcher will be
            # set to a netfilter1
            code = "def host_callback(flow):\n"
            code += "    import os\n"
            code += "    flow.accept = False\n"
            code += f"    os.system(\"iptables -I INPUT -i lo -p tcp --dport {port} -j NFQUEUE --queue-num 1\")\n"
            code += "    pdis.close()\n"
            code += "    pdis.open(\"netfilter1\")\n"
            code += "    pdis.run()\n"
            code += f"dom = pyaiengine.DomainName(\"Local\", \"localhost:{port}\")\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom.callback = host_callback\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"http\")\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            with requests.Session() as inner_session:
                # The first request will pass
                _ = inner_session.get(f"http://localhost:{port}")
                time.sleep(API_DELAY)
                try:
                    _ = inner_session.get(f"http://localhost:{port}", timeout=2)
                    self.assertTrue(False)
                except Exception as exc:
                    self.assertTrue(True)

            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri, headers={"Accept": "application/json"})
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertGreater(jdata["drop_packets"], 5)
            self.assertGreater(jdata["drop_bytes"], 1000)

            code = "dm.show()"
            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Matchs:1", res.text)
            os.system(f"iptables -D INPUT -i lo -p tcp --dport {port} -j NFQUEUE --queue-num 1")

        proc_http.kill()

    def test02(self):
        """Verify that we can drop UDP with netfilter."""

        port = get_free_port()

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            regex_name ="some regex"
            code = "def drop_callback(flow):\n"
            code += "    import os\n"
            code += "    flow.accept = False\n"
            code += f"    os.system(\"iptables -I INPUT -i lo -p udp --dport {port} -j NFQUEUE --queue-num 1\")\n"
            code += "    pdis.close()\n"
            code += "    pdis.open(\"netfilter1\")\n"
            code += "    pdis.run()\n\n"
            code += f"regex = pyaiengine.Regex('{regex_name}', 'AAAA')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.udp_regex_manager = remg\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            sock.sendto(b"AAAA", ("127.0.0.1", port))

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"portdst": port, "protocol": 17}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            flow = jres[0]

            flowid = "[%s:%d]" % (flow["ip"]["src"], flow["port"]["src"])
            flowid += ":%d:" % (flow["proto"])
            flowid += "[%s:%d]" % (flow["ip"]["dst"], flow["port"]["dst"])

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["drop_bytes"], 0)
            self.assertEqual(flow["drop_packets"], 0)

            # Traffic been dropped
            sock.sendto(b"XXXXXXXX", ("127.0.0.1", port))

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["drop_bytes"], 16)
            self.assertEqual(flow["drop_packets"], 1)

            # Set a timer that will revert the netfilter to lo
            code = "def timer_callback():\n"
            code += "    import os\n"
            code += f"    os.system(\"iptables -D INPUT -i lo -p udp --dport {port} -j NFQUEUE --queue-num 1\")\n"
            code += "    pdis.add_timer(None, 1)\n"
            code += "    pdis.close()\n"
            code += "    pdis.open(\"lo\")\n"
            code += "    pdis.run()\n\n"
            code += "pdis.add_timer(timer_callback, 1)\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers, timeout=2)
            self.assertEqual(res.status_code, 200)

            time.sleep(API_DELAY * 2)

            headers = {"Accept": "application/json"}
            uri = f"{self.baseuri}/v1/summary"
            res = session.get(uri, headers=headers)
            self.assertEqual(res.status_code, 200)
            jdata = res.json()
            self.assertEqual(jdata["device"], "lo")

    def test03(self):
        """Verify that we can reject TCP flows by using an
        iptables with reject rule dynamically."""

        port = get_free_port()

        proc_http = Process(target=http_server, args=(port,))
        proc_http.daemon = True
        proc_http.start()

        time.sleep(API_DELAY)

        with requests.Session() as session:

            # Set a callback that when a flow matchs it will create
            # an iptables rule dynamically and the engine will start
            # to drop the packets as the packet dispatcher will be
            # set to a netfilter1
            code = "def host_callback(flow):\n"
            code += "    import os\n"
            code += "    cmd = f\"iptables -I INPUT -i lo -p tcp "
            code += f" --dport {port} -j REJECT --reject-with tcp-reset\"\n"
            code += f"    os.system(cmd)\n\n"
            code += f"dom = pyaiengine.DomainName(\"Local\", \"localhost:{port}\")\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom.callback = host_callback\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"http\")\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            with requests.Session() as inner_session:
                # The first request will pass
                _ = inner_session.get(f"http://localhost:{port}")
                time.sleep(API_DELAY)
                try:
                    _ = inner_session.get(f"http://localhost:{port}", timeout=4)
                    self.assertTrue(False)
                except (ConnectionResetError, Exception) as exc:
                    self.assertTrue(True)

            # Get the flows in JSON format
            headers = {"Accept" : "application/json"}
            params = {"portdst": port, "protocol": 6}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            self.assertEqual(jres, None)

            os.system(f"iptables -D INPUT -i lo -p tcp --dport {port} -j REJECT --reject-with tcp-reset")

        proc_http.kill()

    def test04(self):
        """Verify that we can reject TCP flows by using an
        iptables with reject rule dynamically."""

        port = get_free_port()
        proc_udp = Process(target=udp_server, args=(port,))
        proc_udp.daemon = True
        proc_udp.start()

        time.sleep(API_DELAY)

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            # Set a callback that when a flow matchs it will create
            # an iptables rule dynamically.
            regex_name ="some udp regex"
            code = "def drop_callback(flow):\n"
            code += "    import os\n"
            code += "    cmd = f\"iptables -I INPUT -i lo -p udp "
            code += f" --dport {port} -j REJECT --reject-with icmp-port-unreachable\"\n"
            code += f"    os.system(cmd)\n\n"
            code += f"regex = pyaiengine.Regex('{regex_name}', 'AAAA')\n"
            code += "regex.callback = drop_callback\n"
            code += "remg = pyaiengine.RegexManager()\n"
            code += "remg.add_regex(regex)\n"
            code += "stack.udp_regex_manager = remg\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            sock.sendto(b"PINGAAAA", ("127.0.0.1", port))

            time.sleep(API_DELAY)

            sock.sendto(b"PING", ("127.0.0.1", port))

            params = {"name": "icmp", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["dest_unreachables"], 1)

            os.system(f"iptables -D INPUT -i lo -p udp --dport {port} -j REJECT --reject-with icmp-port-unreachable")

        proc_udp.kill()


@unittest.skipIf(not defined("HAVE_REJECT_FLOW") or os.geteuid() != 0, "Test not supported")
class StackLanLiveRejectHTTPServerTests(unittest.TestCase):
    """Verify the functionality of the HTTP interface
    with live traffic works if the user decide to reset some TCP/UDP flows."""

    @classmethod
    def setUpClass(cls):
        cls.interface = "lo"
        cls.port = get_free_port()
        cls.ipport = get_free_port()
        cls.baseuri = f"http://{HTTP_LISTEN_IP}:{cls.port}"
        generate_service_code("server_code.py", cls.__name__, "StackLan", cls.port,
            f"{cls.interface}")

    def setUp(self):
        """Create a new process with the server code"""

        self.proc = subprocess.Popen([sys.executable, "server_code.py"])
        time.sleep(API_DELAY)

    def tearDown(self):
        # Stop and delete the server
        self.proc.kill()
        self.proc.wait()

    def test01(self):
        """Reset the TCP connection via callback."""

        port = get_free_port()

        proc_http = Process(target=http_server, args=(port,))
        proc_http.daemon = True
        proc_http.start()

        time.sleep(API_DELAY)

        with requests.Session() as session:

            # Set a callback and reset the flow
            code = "def host_callback(flow):\n"
            code += "    flow.reject = True\n\n"
            code += f"dom = pyaiengine.DomainName(\"Local\", \"localhost:{port}\")\n"
            code += "dm = pyaiengine.DomainNameManager()\n"
            code += "dom.callback = host_callback\n"
            code += "dm.add_domain_name(dom)\n"
            code += "stack.set_domain_name_manager(dm, \"http\")\n"

            headers = {"Content-Type" : "text/python"}
            uri = f"{self.baseuri}/v1/python_code"
            res = session.post(uri, data=code, headers=headers)
            self.assertEqual(res.status_code, 200)

            with requests.Session() as inner_session:
                try:
                    res = inner_session.get(f"http://localhost:{port}", timeout=2)
                    time.sleep(1)
                    res = inner_session.get(f"http://localhost:{port}", timeout=2)
                except Exception as exc:
                    print(str(exc))
                    pass

            params = {"name": "tcp", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["flags"]["rsts"], 2)

        proc_http.kill()

    def test02(self):
        """Reset the TCP connection via Rest API."""

        port = get_free_port()

        proc_http = Process(target=http_server, args=(port,))
        proc_http.daemon = True
        proc_http.start()

        time.sleep(API_DELAY)

        with requests.Session() as session, \
            requests.Session() as inner_session:

            res = inner_session.get(f"http://localhost:{port}")
            self.assertEqual(res.status_code, 200)

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            params = {"portdst": port, "protocol": 6}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            flow = jres[0]

            flowid = "[%s:%d]" % (flow["ip"]["src"], flow["port"]["src"])
            flowid += ":%d:" % (flow["proto"])
            flowid += "[%s:%d]" % (flow["ip"]["dst"], flow["port"]["dst"])

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            data = {"reject": True}
            headers = {"Accept" : "application/json"}
            res = session.put(uri, json=data, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, json=data, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["reject"], True)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, json=data, headers={}, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Reject:                        yes", res.text) 

            try:
                res = inner_session.get(f"http://localhost:{port}", timeout=2)
            except Exception as exc:
                print(str(exc))    

            params = {"name": "tcp", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["flags"]["rsts"], 2)

        proc_http.kill()

    def test03(self):
        """Reset the UDP connection via Rest API."""

        port = get_free_port()

        proc_udp = Process(target=udp_server, args=(port,))
        proc_udp.daemon = True
        proc_udp.start()

        time.sleep(API_DELAY)

        with requests.Session() as session, \
            socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:

            sock.sendto(b"PING", ("127.0.0.1", port))
            msg, _ = sock.recvfrom(1024)
            self.assertEqual(msg, b"PONG")

            # Get the flow in JSON format
            headers = {"Accept" : "application/json"}
            params = {"portdst": port, "protocol": 17}
            uri = f"{self.baseuri}/v1/flows"
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            jres = res.json()
            flow = jres[0]
            self.assertEqual(flow["downstream"]["bytes"], 4)
            self.assertEqual(flow["downstream"]["packets"], 1)
            self.assertEqual(flow["upstream"]["bytes"], 4)
            self.assertEqual(flow["upstream"]["packets"], 1)

            flowid = "[%s:%d]" % (flow["ip"]["src"], flow["port"]["src"])
            flowid += ":%d:" % (flow["proto"])
            flowid += "[%s:%d]" % (flow["ip"]["dst"], flow["port"]["dst"])

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            data = {"reject": True}
            headers = {"Accept" : "application/json"}
            res = session.put(uri, json=data, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, json=data, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            flow = res.json()
            self.assertEqual(flow["reject"], True)

            sock.sendto(b"PING", ("127.0.0.1", port))
            msg, _ = sock.recvfrom(1024)
            self.assertEqual(msg, b"PONG")

            params = {"id": flowid}
            uri = f"{self.baseuri}/v1/flow"
            res = session.get(uri, json=data, headers={}, params=params)
            self.assertEqual(res.status_code, 200)
            self.assertIn("Reject:                        yes", res.text) 

            sock.sendto(b"PING", ("127.0.0.1", port))
            params = {"name": "icmp", "stats_level": 4}
            uri = f"{self.baseuri}/v1/protocol"
            headers = {"Accept" : "application/json"}
            res = session.get(uri, headers=headers, params=params)
            self.assertEqual(res.status_code, 200)
            data = res.json()
            self.assertEqual(data["dest_unreachables"], 2)

        proc_udp.kill()


if __name__ == "__main__":

    HTTP_LISTEN_IP = os.getenv("HTTP_LISTEN_IP", "127.0.0.1")
    AUTHORIZED_HOSTS = get_available_ip_addresses()

    clean_log_files()

    if len(sys.argv) > 1 and sys.argv[1] == "-s":
        # User wants to see the available suites
        print("Available tests suites")
        template = "{0:50}{1:<8}"
        header = ("Test suite","Tests")
        print(template.format(*header))
        total_tests = 0
        for name, obj in inspect.getmembers(sys.modules[__name__]):
            if inspect.isclass(obj) and issubclass(obj, unittest.TestCase):
                method_test_list = [func for func in dir(obj) if (callable(getattr(obj, func)) and func.startswith("test"))]
                items = (obj.__name__, len(method_test_list))
                total_tests += len(method_test_list)
                print(template.format(*items))
        items = ("Total", total_tests)
        print(template.format(*items))

        sys.exit(0)

    sys.exit(unittest.main())
